feat:

1、selenium 整合 2、science 网站

feat:
1、selenium 整合 2、science 网站
谢茂盛
1 parent b7789621
Showing 6 changed files with 60 additions and 27 deletions
src/main/java/com/canrd/webmagic/config/SeleniumConfig.java
src/main/java/com/canrd/webmagic/controller/Science4JournalController.java
src/main/java/com/canrd/webmagic/processor/Science4JournalArticlePageProcessor.java
src/main/java/com/canrd/webmagic/processor/Science4JournalSearchPageProcessor.java
src/main/java/com/canrd/webmagic/processor/download/SeleniumDownloader.java
src/main/resources/user-agent/User-Agents.txt
 package com.canrd.webmagic.config;
-import com.canrd.webmagic.processor.config.Agent;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
 import org.openqa.selenium.chrome.ChromeOptions;
@@ -16,9 +15,25 @@ import org.springframework.context.annotation.Configuration;
 @Configuration
 public class SeleniumConfig {
-    @Bean
-    public WebDriver webDriver() {
+//    @Bean
+    public WebDriver webDriver() throws InterruptedException {
         System.setProperty("webdriver.chrome.driver", "D:\\chrome\\chromedriver-win64\\chromedriver-win64\\chromedriver.exe");
-        return new ChromeDriver();
+        // 初始化ChromeOptions
+        ChromeOptions options = new ChromeOptions();
+        // 添加代理，这里使用的代理是示例，需要替换为实际的代理服务器地址和端口
+//        options.addArguments("--proxy-server=http://proxy-server:port");
+
+        // 禁用JavaScript，有时这能帮助绕过Cloudflare的检查
+        options.addArguments("--disable-javascript");
+
+        // 禁用浏览器扩展，如果知道Cloudflare使用了特定的扩展，可以禁用它
+        options.addArguments("--disable-extensions");
+
+        // 禁用本地缓存，确保每次访问都从服务器获取
+        options.addArguments("--disable-application-cache");
+
+        options.setBinary("D:\\chrome\\chrome-win64\\chrome-win64\\chrome.exe");
+
+        return new ChromeDriver(options);
     }
 }
@@ -5,7 +5,9 @@ import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
 import com.canrd.webmagic.processor.Science4JournalSearchPageProcessor;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
 import com.canrd.webmagic.service.ArticleService;
+import org.apache.logging.log4j.core.util.UuidUtil;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import us.codecraft.webmagic.Spider;
@@ -30,6 +32,9 @@ public class Science4JournalController {
     @Resource
     private Science4JournalSearchPageProcessor science4JournalSearchPageProcessor;
+    @Resource
+    private SeleniumDownloader seleniumDownloader;
+
     /**
      * @return
      */
@@ -37,8 +42,10 @@ public class Science4JournalController {
     public ServerResult start(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "indexSize") Integer indexSize) {
         for (int i = 0; i <= indexSize; i++) {
             Spider.create(science4JournalSearchPageProcessor)
-                    .addUrl("http://www.science.org/journal/science/insights?startPage=" + i)
+                    .addUrl("https://www.science.org/journal/science/insights?startPage=" + i)
                     // 开启5个线程执行，并开始爬取
+                    .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                    .setDownloader(seleniumDownloader)
                     .thread(5).run();
         }
@@ -32,10 +32,9 @@ import java.util.Objects;
 @Slf4j
 @Component
 public class Science4JournalArticlePageProcessor implements PageProcessor {
-    private String agent = Agent.getRandom();
     // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(Agent.getRandom());
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(5).setUserAgent(Agent.getRandom());
     /**
      * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
@@ -108,6 +107,10 @@ public class Science4JournalArticlePageProcessor implements PageProcessor {
         return site;
     }
+    public void setSite(Site site) {
+        this.site = site;
+    }
+
     public static void main(String[] args) {
         // 创建一个Spider，并把我们的处理器放进去
         Spider.create(new Science4JournalArticlePageProcessor())
@@ -6,6 +6,7 @@ import com.canrd.webmagic.processor.config.Agent;
 import com.canrd.webmagic.processor.download.SeleniumDownloader;
 import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.core.util.UuidUtil;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
@@ -43,7 +44,7 @@ public class Science4JournalSearchPageProcessor implements PageProcessor {
     /**
      * 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
      */
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(Agent.getRandom());
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(5).setUserAgent(Agent.getRandom());
     /**
      * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
@@ -84,6 +85,7 @@ public class Science4JournalSearchPageProcessor implements PageProcessor {
                                 .addUrl(link)
                                 .addPipeline(articlePipeline)
                                 .setDownloader(seleniumDownloader)
+                                .setUUID(UuidUtil.getTimeBasedUuid().toString())
                                 // 开启5个线程执行，并开始爬取
                                 .thread(1).run();
                         log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
 package com.canrd.webmagic.processor.download;
+import com.canrd.webmagic.config.SeleniumConfig;
+import com.canrd.webmagic.processor.config.Agent;
 import lombok.extern.slf4j.Slf4j;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Cookie;
@@ -26,10 +28,10 @@ import java.util.Map;
 @Slf4j
 @Component
 public class SeleniumDownloader extends AbstractDownloader {
-    private int sleepTime = 0;
+    private int sleepTime = 30;
     @Resource
-    private WebDriver webDriver;
+    private SeleniumConfig config;
     /**
      * set sleep time to wait until load success
@@ -45,20 +47,12 @@ public class SeleniumDownloader extends AbstractDownloader {
     @Override
     public Page download(Request request, Task task) {
         Page page = Page.fail();
+        WebDriver webDriver = null;
         try {
-
-
-            log.info("downloading page " + request.getUrl());
-            webDriver.get(request.getUrl());
-            try {
-                if (sleepTime > 0) {
-                    Thread.sleep(sleepTime);
-                }
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
+            webDriver = config.webDriver();
             WebDriver.Options manage = webDriver.manage();
             Site site = task.getSite();
+            site.setUserAgent(Agent.getRandom());
             if (site.getCookies() != null) {
                 for (Map.Entry<String, String> cookieEntry : site.getCookies()
                         .entrySet()) {
@@ -68,6 +62,18 @@ public class SeleniumDownloader extends AbstractDownloader {
                 }
             }
+            log.info("downloading page " + request.getUrl());
+
+            webDriver.get(request.getUrl());
+            try {
+                if (sleepTime > 0) {
+                    Thread.sleep(sleepTime);
+                }
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+
+
             /*
              * TODO You can add mouse event or other processes
              *
@@ -75,7 +81,7 @@ public class SeleniumDownloader extends AbstractDownloader {
              */
             try {
                 //休眠3秒就是为了动态的数据渲染完成后在进行获取
-                Thread.sleep(30000);
+                Thread.sleep(3000);
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -91,7 +97,11 @@ public class SeleniumDownloader extends AbstractDownloader {
             log.warn("download page {} error", request.getUrl(), e);
             onError(request, task, e);
         } finally {
-
+            if (webDriver != null) {
+                webDriver.close();
+                webDriver.quit();
+                webDriver = null;
+            }
         }
         return page;
     }
@@ -3,19 +3,15 @@ Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera
 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50
 Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2
-Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11
 Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36
 Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER
-Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)
 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"
-Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)
 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
-Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36