matter官网

凌世锦
1 parent b17c8d02
Showing 15 changed files with 102 additions and 55 deletions
src/main/java/com/canrd/webmagic/controller/MatterController.java
src/main/java/com/canrd/webmagic/driver/ChromeBuildDriver.java
src/main/java/com/canrd/webmagic/processor/ChemicalPagePcoessor.java
src/main/java/com/canrd/webmagic/processor/MatterPragePcoessor.java
src/main/java/com/canrd/webmagic/processor/driver/MyChromeDriver.java
target/classes/application-local.yml
target/classes/com/canrd/webmagic/config/SeleniumConfig.class
target/classes/com/canrd/webmagic/controller/MatterController.class
target/classes/com/canrd/webmagic/processor/ChemicalPagePcoessor.class
target/classes/com/canrd/webmagic/processor/MatterPragePcoessor.class
target/classes/com/canrd/webmagic/processor/download/SeleniumDownloader.class
target/classes/com/canrd/webmagic/processor/driver/BrowserDriver.class
target/classes/com/canrd/webmagic/processor/driver/MyChromeDriver.class
target/classes/com/canrd/webmagic/processor/driver/MyEdgeDriver.class
target/classes/com/canrd/webmagic/processor/driver/MyFirefoxDriver.class
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.MatterPragePcoessor;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/matter/article")
+@Api("Matter")
+public class MatterController {
+    @Resource
+    private MatterPragePcoessor matterPragePcoessor;
+
+    @Resource
+    private SeleniumDownloader seleniumDownloader;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start()  {
+        Spider.create(new MatterPragePcoessor())
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.cell.com/matter/fulltext/S2590-2385(21)00631-7#%20")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .setDownloader(seleniumDownloader.setSleepTime(30000))
+                // 开启5个线程执行，并开始爬取
+                .thread(5).run();
+        return ServerResult.success();
+    }
+}
-package com.canrd.webmagic.driver;
-
-import org.openqa.selenium.chrome.ChromeOptions;
-
-import java.util.Arrays;
-
-public class ChromeBuildDriver {
-    public ChromeOptions build(String DriverPath){
-        ChromeOptions options = new ChromeOptions();
-        // 添加代理，这里使用的代理是示例，需要替换为实际的代理服务器地址和端口
-//        options.addArguments("--proxy-server=http://proxy-server:port");
-
-        // 禁用JavaScript，有时这能帮助绕过Cloudflare的检查
-        options.addArguments("--disable-javascript");
-
-        // 禁用浏览器扩展，如果知道Cloudflare使用了特定的扩展，可以禁用它
-        options.addArguments("--disable-extensions");
-
-        // 禁用本地缓存，确保每次访问都从服务器获取
-        options.addArguments("--disable-application-cache");
-
-        // 禁止策略化
-        options.addArguments("--disable-infobars");
-        // 解决DevToolsActivePort文件不存在的报错
-        options.addArguments("--no-sandbox");
-        // 指定浏览器分辨
-        options.addArguments("window-size=1920x3000");
-        // 谷歌文档提到需要加上这个属性来规避bug
-        options.addArguments("--disable-gpu");
-        // 隐身模式（无痕模式）
-        options.addArguments("--incognito");
-        // 最大化运行（全屏窗口）,不设置，取元素会报错
-        options.addArguments("--start-maximized");
-        // 禁用浏览器正在被自动化程序控制的提示
-        options.addArguments("--disable-infobars");
-        // 隐藏滚动条, 应对一些特殊页面
-        options.addArguments("--hide-scrollbars");
-        // 不加载图片, 提升速度
-        options.addArguments("blink-settings=imagesEnabled=false");
-        // 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
-        options.addArguments("--headless");
-        //禁用 blink 特征
-        options.addArguments("disable-blink-features=AutomationControlled");
-        options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
-        options.setExperimentalOption("useAutomationExtension", false);
-        options.addArguments("--remote-allow-origins=*");
-        options.setBinary(DriverPath);
-        return options;
-    }
-}
 package com.canrd.webmagic.processor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
 import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
 public class ChemicalPagePcoessor implements PageProcessor {
-
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
     @Override
     public void process(Page page) {
+    };
+
+    public void doArticleContent(Page page){
+        Html html = page.getHtml();
+
+    }
+
+    public static void main(String[] args) {
+        // 创建一个Spider，并把我们的处理器放进去
+        Spider.create(new ChemicalPagePcoessor())
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=1")
+                .addPipeline(new ArticlePipeline())
+                // 开启5个线程执行，并开始爬取
+                .thread(5).run();
     }
 }
+package com.canrd.webmagic.processor;
+
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+@Component
+public class MatterPragePcoessor implements PageProcessor {
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
+    @Override
+    public void process(Page page) {
+
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    public void doArticleContent(Page page){
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+//        html.xpath()
+    }
+
+    public static void main(String[] args) {
+        // 创建一个Spider，并把我们的处理器放进去
+        Spider.create(new MatterPragePcoessor())
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.cell.com/matter/fulltext/S2590-2385(21)00631-7#%20")
+                .addPipeline(new ArticlePipeline())
+                // 开启5个线程执行，并开始爬取
+                .thread(5).run();
+    }
+}
@@ -16,7 +16,9 @@ import java.util.Arrays;
  */
 @Component
 public class MyChromeDriver implements BrowserDriver{
-    private static final String WIN_DRIVER_PATH = "D:\\driver\\chrome\\chromedriver-win64\\chromedriver.exe";
+//    private static final String WIN_DRIVER_PATH = "D:\\driver\\chrome\\chromedriver-win64\\chromedriver.exe";
+
+    private static final String WIN_DRIVER_PATH = "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe";
     private static final String LINUX_DRIVER_PATH = "/home/canrd/webmagic/chrome/chromedriver-linux64/chromedriver";
     private static final String WIN_BINARY_PATH = "D:\\driver\\chrome\\chrome-headless-shell-win64\\chrome-headless-shell.exe";
     private static final String LINUX_BINARY_PATH = "/home/canrd/webmagic/chrome/chrome-linux64/chrome";
@@ -57,13 +57,13 @@ spring:
     testWhileIdle: true
     testOnBorrow: true
     testOnReturn: true
-    password: canrd@2024
+    password: 123456
     time-between-eviction-runs-millis: 1000
-    url: jdbc:mysql://39.108.227.113:3307/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
+    url: jdbc:mysql://localhost:3306/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root
   redis:
     database: 0
-    host: 39.108.227.113
+    host: localhost
     lettuce:
       pool:
         max-active: 2000