新增了火狐和edge驱动

zhongnanhuang
1 parent f7957dbb
Showing 8 changed files with 240 additions and 59 deletions
pom.xml
src/main/java/com/canrd/webmagic/config/SeleniumConfig.java
src/main/java/com/canrd/webmagic/processor/download/SeleniumDownloader.java
src/main/java/com/canrd/webmagic/processor/driver/BrowserDriver.java
src/main/java/com/canrd/webmagic/processor/driver/MyChromeDriver.java
src/main/java/com/canrd/webmagic/processor/driver/MyEdgeDriver.java
src/main/java/com/canrd/webmagic/processor/driver/MyFirefoxDriver.java
src/main/resources/application-local.yml
@@ -48,6 +48,7 @@
         <easyexcel.version>2.2.3</easyexcel.version>
         <webmagic.version>0.10.0</webmagic.version>
         <selenium.version>3.4.0</selenium.version>
+<!--        <browsermob.version>2.1.5</browsermob.version>-->
     </properties>
     <dependencies>
@@ -63,6 +64,12 @@
                 </exclusion>
             </exclusions>
         </dependency>
+<!--        <dependency>-->
+<!--            <groupId>net.lightbody.bmp</groupId>-->
+<!--            <artifactId>browsermob-core</artifactId>-->
+<!--            <version>${browsermob.version}</version>-->
+<!--        </dependency>-->
+
         <!-- webmagic核心库 -->
         <dependency>
 package com.canrd.webmagic.config;
+import com.canrd.webmagic.processor.driver.BrowserDriver;
+import com.canrd.webmagic.processor.driver.MyChromeDriver;
+import com.canrd.webmagic.processor.driver.MyFirefoxDriver;
+import lombok.Data;
 import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeOptions;
-import org.springframework.context.annotation.Bean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
 import org.springframework.context.annotation.Configuration;
-import java.util.Arrays;
+import java.util.*;
+
 /**
  * @author: xms
@@ -15,62 +20,32 @@ import java.util.Arrays;
  * @version: 1.0
  */
 @Configuration
-public class SeleniumConfig {
+@Data
+public class SeleniumConfig implements ApplicationContextAware {
-//    @Bean
-    public WebDriver webDriver() throws InterruptedException {
-        // 初始化ChromeOptions
-        ChromeOptions options = new ChromeOptions();
-        // 添加代理，这里使用的代理是示例，需要替换为实际的代理服务器地址和端口
-//        options.addArguments("--proxy-server=http://proxy-server:port");
+    private static ApplicationContext context;
-        // 禁用JavaScript，有时这能帮助绕过Cloudflare的检查
-        options.addArguments("--disable-javascript");
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) {
+        context = applicationContext;
+    }
-        // 禁用浏览器扩展，如果知道Cloudflare使用了特定的扩展，可以禁用它
-        options.addArguments("--disable-extensions");
+    public static final String FIREFOX_KEY = "myFirefoxDriver";
+    public static final String CHROME_KEY = "myChromeDriver";
+    public static final String EDGE_KEY = "myEdgeDriver";
-        // 禁用本地缓存，确保每次访问都从服务器获取
-        options.addArguments("--disable-application-cache");
+    public static final List<String> DRIVER_KEYS = new ArrayList<>(Arrays.asList(
+            CHROME_KEY,
+            FIREFOX_KEY,
+            EDGE_KEY
+    ));
-        // 禁止策略化
-        options.addArguments("--disable-infobars");
-        // 解决DevToolsActivePort文件不存在的报错
-        options.addArguments("--no-sandbox");
-        // 指定浏览器分辨
-        options.addArguments("window-size=1920x3000");
-        // 谷歌文档提到需要加上这个属性来规避bug
-        options.addArguments("--disable-gpu");
-        // 隐身模式（无痕模式）
-        options.addArguments("--incognito");
-        // 最大化运行（全屏窗口）,不设置，取元素会报错
-        options.addArguments("--start-maximized");
-        // 禁用浏览器正在被自动化程序控制的提示
-        options.addArguments("--disable-infobars");
-        // 隐藏滚动条, 应对一些特殊页面
-        options.addArguments("--hide-scrollbars");
-        // 不加载图片, 提升速度
-        options.addArguments("blink-settings=imagesEnabled=false");
-        // 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
-        options.addArguments("--headless");
-        //禁用 blink 特征
-        options.addArguments("disable-blink-features=AutomationControlled");
-        options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
-        options.setExperimentalOption("useAutomationExtension", false);
-        options.addArguments("--remote-allow-origins=*");
-        String os_name = System.getProperty("os.name");
-        // 判断是否是windows系统
-        if (os_name.toLowerCase().startsWith("win")) {
-            // windows
-            options.setBinary("D:\\chrome\\chrome-headless-shell-win64\\chrome-headless-shell-win64\\chrome-headless-shell.exe");
-            System.setProperty("webdriver.chrome.driver", "D:\\chrome\\chromedriver-win64\\chromedriver-win64\\chromedriver.exe");
-        } else {
-            // linux
-            options.setBinary("/home/canrd/webmagic/chrome/chrome-linux64/chrome");
-            System.setProperty("webdriver.chrome.driver", "/home/canrd/webmagic/chrome/chromedriver-linux64/chromedriver");
+    public WebDriver getWebDriver(String key) {
+        Object bean = context.getBean(key);
+        if (bean instanceof BrowserDriver) {
+            return ((BrowserDriver) bean).getDriver();
         }
-
-        return new ChromeDriver(options);
+        return null;
     }
 }
@@ -49,7 +49,8 @@ public class SeleniumDownloader extends AbstractDownloader {
         Page page = Page.fail();
         WebDriver webDriver = null;
         try {
-            webDriver = config.webDriver();
+            //0默认谷歌浏览器
+            webDriver = config.getWebDriver(SeleniumConfig.DRIVER_KEYS.get(0));
             WebDriver.Options manage = webDriver.manage();
             Site site = task.getSite();
             site.setUserAgent(Agent.getRandom());
@@ -88,7 +89,7 @@ public class SeleniumDownloader extends AbstractDownloader {
         } finally {
             if (webDriver != null) {
                 webDriver.close();
-                webDriver.quit();
+//                webDriver.quit();
                 webDriver = null;
             }
         }
+package com.canrd.webmagic.processor.driver;
+
+import org.openqa.selenium.WebDriver;
+
+/**
+ * @author zhongnanhuang
+ * @version 1.0
+ * @project webmagic-canrd-service
+ * @description 驱动接口
+ * @date 2024/5/23 10:46:16
+ */
+public interface BrowserDriver {
+
+    WebDriver getDriver();
+
+}
+package com.canrd.webmagic.processor.driver;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+
+/**
+ * @author zhongnanhuang
+ * @version 1.0
+ * @project webmagic-canrd-service
+ * @description 谷歌浏览器驱动
+ * @date 2024/5/23 10:49:30
+ * 驱动下载地址：https://storage.googleapis.com/chrome-for-testing-public
+ */
+@Component
+public class MyChromeDriver implements BrowserDriver{
+    private static final String WIN_DRIVER_PATH = "D:\\driver\\chrome\\chromedriver-win64\\chromedriver.exe";
+    private static final String LINUX_DRIVER_PATH = "/home/canrd/webmagic/chrome/chromedriver-linux64/chromedriver";
+    private static final String WIN_BINARY_PATH = "D:\\driver\\chrome\\chrome-headless-shell-win64\\chrome-headless-shell.exe";
+    private static final String LINUX_BINARY_PATH = "/home/canrd/webmagic/chrome/chrome-linux64/chrome";
+
+    @Override
+    public WebDriver getDriver() {
+        // 初始化ChromeOptions
+        ChromeOptions options = new ChromeOptions();
+        // 添加代理，这里使用的代理是示例，需要替换为实际的代理服务器地址和端口
+//        options.addArguments("--proxy-server=http://proxy-server:port");
+
+        // 禁用JavaScript，有时这能帮助绕过Cloudflare的检查
+        options.addArguments("--disable-javascript");
+
+        // 禁用浏览器扩展，如果知道Cloudflare使用了特定的扩展，可以禁用它
+        options.addArguments("--disable-extensions");
+
+        // 禁用本地缓存，确保每次访问都从服务器获取
+        options.addArguments("--disable-application-cache");
+
+        // 禁止策略化
+        options.addArguments("--disable-infobars");
+        // 解决DevToolsActivePort文件不存在的报错
+        options.addArguments("--no-sandbox");
+        // 指定浏览器分辨
+        options.addArguments("window-size=1920x3000");
+        // 谷歌文档提到需要加上这个属性来规避bug
+        options.addArguments("--disable-gpu");
+        // 隐身模式（无痕模式）
+        options.addArguments("--incognito");
+        // 最大化运行（全屏窗口）,不设置，取元素会报错
+        options.addArguments("--start-maximized");
+        // 禁用浏览器正在被自动化程序控制的提示
+        options.addArguments("--disable-infobars");
+        // 隐藏滚动条, 应对一些特殊页面
+        options.addArguments("--hide-scrollbars");
+        // 不加载图片, 提升速度
+        options.addArguments("blink-settings=imagesEnabled=false");
+        // 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
+        options.addArguments("--headless");
+        //禁用 blink 特征
+        options.addArguments("disable-blink-features=AutomationControlled");
+        options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
+        options.setExperimentalOption("useAutomationExtension", false);
+        options.addArguments("--remote-allow-origins=*");
+
+
+        String os_name = System.getProperty("os.name");
+        // 判断是否是windows系统
+        if (os_name.toLowerCase().startsWith("win")) {
+            // windows
+            options.setBinary(WIN_BINARY_PATH);
+            System.setProperty("webdriver.chrome.driver", WIN_DRIVER_PATH);
+        } else {
+            // linux
+            options.setBinary(LINUX_BINARY_PATH);
+            System.setProperty("webdriver.chrome.driver", LINUX_DRIVER_PATH);
+        }
+
+        return new org.openqa.selenium.chrome.ChromeDriver(options);
+    }
+}
+package com.canrd.webmagic.processor.driver;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.edge.EdgeDriver;
+import org.openqa.selenium.edge.EdgeOptions;
+import org.openqa.selenium.firefox.FirefoxOptions;
+import org.openqa.selenium.firefox.FirefoxProfile;
+import org.springframework.stereotype.Component;
+
+import java.time.Duration;
+
+/**
+ * @author zhongnanhuang
+ * @version 1.0
+ * @project webmagic-canrd-service
+ * @description 火狐浏览器驱动
+ * @date 2024/5/23 10:45:50
+ * 驱动下载地址：https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/?form=MA13LH#downloads
+ */
+@Component
+public class MyEdgeDriver implements BrowserDriver{
+    private static final String WIN_DRIVER_PATH = "D:\\driver\\edge\\msedgedriver.exe";
+    private static final String LINUX_DRIVER_PATH = "/home/canrd/webmagic/chrome/chromedriver-linux64/chromedriver";
+
+    @Override
+    public WebDriver getDriver() {
+        // 设置 Chrome 驱动程序路径
+        System.setProperty("webdriver.edge.driver", WIN_DRIVER_PATH); // 替换为实际的驱动程序路径
+
+        // 初始化 ChromeOptions 对象
+        EdgeOptions options = new EdgeOptions();
+//        options.setPageLoadTimeout(Duration.ofSeconds(30)); // 设置页面加载超时时间为30秒
+//        options.addArguments("--blink-settings=imagesEnabled=false"); // 禁用图片加载
+//        options.addArguments("--disable-notifications"); // 禁用浏览器通知
+//        options.addArguments("--disable-infobars"); // 禁用信息栏
+//        options.addArguments("--disable-extensions"); // 禁用扩展
+//        options.addArguments("--disable-dev-shm-usage"); // 禁用/dev/shm使用
+//        options.setHeadless(true);
+
+
+        // 设置其他选项，例如添加扩展程序等
+        // 初始化 WebDriver
+        EdgeDriver driver = new EdgeDriver(options);
+
+        return driver;
+    }
+}
+package com.canrd.webmagic.processor.driver;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.firefox.FirefoxOptions;
+import org.openqa.selenium.firefox.FirefoxProfile;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author zhongnanhuang
+ * @version 1.0
+ * @project webmagic-canrd-service
+ * @description 火狐浏览器驱动
+ * @date 2024/5/23 10:45:50
+ * 驱动下载地址：https://objects.githubusercontent.com/github-production-release-asset-2e65be
+ */
+@Component
+public class MyFirefoxDriver implements BrowserDriver{
+    private static final String WIN_DRIVER_PATH = "D:\\driver\\firefox\\geckodriver.exe";
+    private static final String LINUX_DRIVER_PATH = "/home/canrd/webmagic/chrome/chromedriver-linux64/chromedriver";
+
+    @Override
+    public WebDriver getDriver() {
+        // 创建 Firefox 配置文件
+        FirefoxProfile profile = new FirefoxProfile();
+        profile.setPreference("permissions.default.image", 2); // 禁用图片加载
+        profile.setPreference("permissions.default.stylesheet", 2); // 禁用 CSS
+        profile.setPreference("media.peerconnection.enabled", false); // 禁用 WebRTC
+        profile.setPreference("dom.webnotifications.enabled", false); // 禁用通知
+//        profile.setPreference("general.useragent.override", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"); // 设置自定义 User-Agent
+
+        // 创建 Firefox 选项
+        FirefoxOptions options = new FirefoxOptions();
+        options.setProfile(profile);
+
+
+
+        String os_name = System.getProperty("os.name");
+        // 判断是否是windows系统
+        if (os_name.toLowerCase().startsWith("win")) {
+            // windows
+            // 指定 GeckoDriver 的路径
+            System.setProperty("webdriver.gecko.driver", WIN_DRIVER_PATH); // 替换为实际的 geckodriver 路径
+        } else {
+            // linux
+            System.setProperty("webdriver.chrome.driver", LINUX_DRIVER_PATH);
+        }
+
+        // 初始化 WebDriver
+        WebDriver driver = new org.openqa.selenium.firefox.FirefoxDriver(options);
+        return driver;
+    }
+}
@@ -57,13 +57,13 @@ spring:
     testWhileIdle: true
     testOnBorrow: true
     testOnReturn: true
-    password: canrd@2024
+    password: 123456
     time-between-eviction-runs-millis: 1000
-    url: jdbc:mysql://39.108.227.113:3307/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
+    url: jdbc:mysql://localhost:3306/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root
   redis:
     database: 0
-    host: 39.108.227.113
+    host: localhost
     lettuce:
       pool:
         max-active: 2000