feat:

1、selenium 整合 2、science 网站

feat:
1、selenium 整合 2、science 网站
谢茂盛
1 parent 8f304c93
Showing 13 changed files with 557 additions and 19 deletions
pom.xml
src/main/java/com/canrd/webmagic/config/SeleniumConfig.java
src/main/java/com/canrd/webmagic/controller/ArticleController.java
src/main/java/com/canrd/webmagic/controller/Science4JournalController.java
src/main/java/com/canrd/webmagic/domain/ArticleTypeEnum.java
src/main/java/com/canrd/webmagic/job/NatureJob.java
src/main/java/com/canrd/webmagic/processor/NatureSearchPageProcessor.java
src/main/java/com/canrd/webmagic/processor/Science4JournalArticlePageProcessor.java
src/main/java/com/canrd/webmagic/processor/Science4JournalSearchPageProcessor.java
src/main/java/com/canrd/webmagic/processor/config/Downloader.java → src/main/java/com/canrd/webmagic/processor/download/Downloader.java
src/main/java/com/canrd/webmagic/processor/download/SeleniumDownloader.java
src/main/java/com/canrd/webmagic/processor/pipeline/NatureArticlePipeline.java → src/main/java/com/canrd/webmagic/processor/pipeline/ArticlePipeline.java
src/main/resources/user-agent/User-Agents.txt
@@ -47,6 +47,7 @@
         <jjwt.version>0.10.6</jjwt.version>
         <easyexcel.version>2.2.3</easyexcel.version>
         <webmagic.version>0.10.0</webmagic.version>
+        <selenium.version>3.4.0</selenium.version>
     </properties>
     <dependencies>
@@ -78,6 +79,12 @@
             <version>${webmagic.version}</version>
         </dependency>
+        <!-- selenium -->
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>${selenium.version}</version>
+        </dependency>
         <!-- Lombok 依赖-->
         <dependency>
+package com.canrd.webmagic.config;
+
+import com.canrd.webmagic.processor.config.Agent;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/26 14:37
+ * @version: 1.0
+ */
+@Configuration
+public class SeleniumConfig {
+
+    @Bean
+    public WebDriver webDriver() {
+        System.setProperty("webdriver.chrome.driver", "D:\\chrome\\chromedriver-win64\\chromedriver-win64\\chromedriver.exe");
+        return new ChromeDriver();
+    }
+}
@@ -5,8 +5,8 @@ import com.canrd.webmagic.common.jsr303.OperateGroup;
 import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
 import com.canrd.webmagic.domain.vo.NatureArticleVO;
 import com.canrd.webmagic.processor.NatureSearchPageProcessor;
-import com.canrd.webmagic.processor.config.Downloader;
-import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
+import com.canrd.webmagic.processor.download.Downloader;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import com.canrd.webmagic.service.ArticleService;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
@@ -33,7 +33,7 @@ public class ArticleController {
     private NatureSearchPageProcessor natureSearchPageProcessor;
     @Resource
-    private NatureArticlePipeline articlePipeline;
+    private ArticlePipeline articlePipeline;
     @Resource
     private Downloader downloader;
@@ -49,7 +49,7 @@ public class ArticleController {
                     .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + i)
                     .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=" + i)
                     .addPipeline(articlePipeline)
-                    .setDownloader(downloader.newIpDownloader())
+//                    .setDownloader(downloader.newIpDownloader())
                     // 开启5个线程执行，并开始爬取
                     .thread(5).run();
         }
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.common.jsr303.OperateGroup;
+import com.canrd.webmagic.domain.vo.NatureArticleQueryVO;
+import com.canrd.webmagic.domain.vo.NatureArticleVO;
+import com.canrd.webmagic.processor.Science4JournalSearchPageProcessor;
+import com.canrd.webmagic.service.ArticleService;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.*;
+import us.codecraft.webmagic.Spider;
+
+import javax.annotation.Resource;
+
+/**
+ * nature-文章信息(NatureArticle)表控制层
+ *
+ * @author makejava
+ * @since 2024-04-07 18:39:41
+ */
+@RestController
+@RequestMapping("/science/journal")
+public class Science4JournalController {
+    /**
+     * 服务对象
+     */
+    @Resource
+    private ArticleService articleService;
+
+    @Resource
+    private Science4JournalSearchPageProcessor science4JournalSearchPageProcessor;
+
+    /**
+     * @return
+     */
+    @GetMapping("/start")
+    public ServerResult start(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "indexSize") Integer indexSize) {
+        for (int i = 0; i <= indexSize; i++) {
+            Spider.create(science4JournalSearchPageProcessor)
+                    .addUrl("http://www.science.org/journal/science/insights?startPage=" + i)
+                    // 开启5个线程执行，并开始爬取
+                    .thread(5).run();
+        }
+
+        return ServerResult.success();
+    }
+
+    /**
+     * 分页查询
+     *
+     * @param natureArticleQueryVO 查询条件
+     * @return 查询结果
+     */
+    @PostMapping("/list")
+    public ServerResult list(@RequestBody @Validated({OperateGroup.List.class}) NatureArticleQueryVO natureArticleQueryVO) {
+        return articleService.list(natureArticleQueryVO);
+    }
+
+    /**
+     * 通过主键查询单条数据
+     *
+     * @param natureArticleQueryVO 查询条件
+     * @return 单条数据
+     */
+    @PostMapping("/query_by_id")
+    public ServerResult queryById(@RequestBody NatureArticleQueryVO natureArticleQueryVO) {
+        return articleService.queryById(natureArticleQueryVO);
+    }
+
+    /**
+     * 新增数据
+     *
+     * @param natureArticleVO 数据VO
+     * @return 新增结果
+     */
+    @PostMapping("/add")
+    public ServerResult add(@RequestBody NatureArticleVO natureArticleVO) {
+        return articleService.add(natureArticleVO);
+    }
+
+    /**
+     * 编辑数据
+     *
+     * @param natureArticleVO 数据VO
+     * @return 编辑结果
+     */
+    @PostMapping("/edit")
+    public ServerResult edit(@RequestBody NatureArticleVO natureArticleVO) {
+        return articleService.edit(natureArticleVO);
+    }
+
+    /**
+     * 删除数据
+     *
+     * @param natureArticleQueryVO 查询条件
+     * @return 删除是否成功
+     */
+    @PostMapping("/delete_by_id")
+    public ServerResult deleteById(@RequestBody NatureArticleQueryVO natureArticleQueryVO) {
+        return articleService.deleteById(natureArticleQueryVO);
+    }
+
+}
+
@@ -15,6 +15,7 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public enum ArticleTypeEnum {
     NATURE("nature", "nature网址"),
+    SCIENCE("science", "science网址"),
     ;
     private String type;
     private String desc;
+package com.canrd.webmagic.job;
+
+import com.canrd.webmagic.common.utils.KeywordUtil;
+import com.canrd.webmagic.processor.NatureSearchPageProcessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Spider;
+
+import javax.annotation.Resource;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/26 10:06
+ * @version: 1.0
+ */
+@Component
+public class NatureJob {
+
+    @Resource
+    private NatureSearchPageProcessor natureSearchPageProcessor;
+
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    /**
+     * 每天凌晨执行一次
+     */
+//    @Scheduled(cron = "*/20 * * * * ?")
+    @Scheduled(cron = "0 0 0 * * ?")
+    public void executeByDay() {
+        for (String keyword : KeywordUtil.getKeyWordList()) {
+            Spider.create(natureSearchPageProcessor)
+                    // 添加这个Spider要爬取的网页地址
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 1)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 2)
+                    .addUrl("https://www.nature.com/search?q=" + keyword + "&page=" + 3)
+                    .addPipeline(articlePipeline)
+//                    .setDownloader(downloader.newIpDownloader())
+                    // 开启5个线程执行，并开始爬取
+                    .thread(5).run();
+        }
+    }
+}
@@ -3,12 +3,13 @@ package com.canrd.webmagic.processor;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
 import com.canrd.webmagic.common.utils.KeywordUtil;
 import com.canrd.webmagic.common.utils.StringUtils;
 import com.canrd.webmagic.domain.ArticleTypeEnum;
 import com.canrd.webmagic.domain.dto.ArticleDO;
 import com.canrd.webmagic.processor.config.Agent;
-import com.canrd.webmagic.processor.pipeline.NatureArticlePipeline;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
@@ -19,9 +20,9 @@ import us.codecraft.webmagic.selector.Html;
 import us.codecraft.webmagic.selector.Selectable;
 import us.codecraft.webmagic.selector.XpathSelector;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
 import java.util.stream.Collectors;
 /**
@@ -104,6 +105,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         }
         String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
         String publishTime;
+        Date publishTimeDateTime = null;
         try {
             publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
         } catch (Exception e) {
@@ -113,6 +115,13 @@ public class NatureSearchPageProcessor implements PageProcessor {
                 publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
             }
         }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
         Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
         List<Selectable> authorNodes = authorSelectable.nodes();
         StringBuffer authorName = new StringBuffer();
@@ -170,7 +179,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
                 .articleCode(articleCode)
                 .authorName(authorName.toString())
                 .title(title)
-                .publishTime(publishTime)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
                 .emailInfo(authorEmail.toJSONString())
                 .articleDesc(articleDesc)
                 .authorAddress(authorAddress.toJSONString())
@@ -191,14 +200,25 @@ public class NatureSearchPageProcessor implements PageProcessor {
         /**
          * 获取到指定的dom后，从这些dom中提取元素内容。
          */
-        for (int i = 1; i <= nodes.size() - 1; i++) {
+        for (int i = 0; i <= nodes.size() - 1; i++) {
             Selectable node = nodes.get(i).$(".u-full-height").nodes().get(2).nodes().get(0).$(".u-full-height").select(new XpathSelector("a[@class='c-card__link u-link-inherit']")).nodes().get(0);
-            String link = node.$("a", "href").get();
+            String link = node.links().get();
             String title = node.$("a", "text").get();
             if (KeywordUtil.containKeywordsInTitle(title)) {
-                page.addTargetRequest(link);
-                log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
+                String publishTime = nodes.get(i).xpath("//div[@class='c-card__section c-meta']/time/text()").get();
+                SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+                try {
+                    Date publishTimeDateTime = formatter.parse(publishTime);
+                    if (!publishTimeDateTime.before(DateUtil.localDate2Date(DateUtil.parseDate("2000-01-01", DateUtil.DATE)))) {
+                        page.addTargetRequest(link);
+                        log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
+                    }
+                } catch (ParseException e) {
+                    e.printStackTrace();
+                }
+
             }
+
         }
     }
@@ -212,7 +232,7 @@ public class NatureSearchPageProcessor implements PageProcessor {
         Spider.create(new NatureSearchPageProcessor())
                 // 添加这个Spider要爬取的网页地址
                 .addUrl("https://www.nature.com/nature/research-articles?sort=PubDate&page=1")
-                .addPipeline(new NatureArticlePipeline())
+                .addPipeline(new ArticlePipeline())
                 // 开启5个线程执行，并开始爬取
                 .thread(5).run();
     }
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.config.Agent;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/1 14:19
+ * @version: 1.0
+ */
+@Slf4j
+@Component
+public class Science4JournalArticlePageProcessor implements PageProcessor {
+    private String agent = Agent.getRandom();
+
+    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(Agent.getRandom());
+
+    /**
+     * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
+     *
+     * @param page
+     */
+    @Override
+    public void process(Page page) {
+        doArticleContent(page);
+    }
+
+    /**
+     * @param page
+     */
+    private void doArticleContent(Page page) {
+
+        //解析页面
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='article-container']/article/header");
+
+        String title = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='core-lede']/div/text()").get();
+
+        String articleDesc = html.xpath("//div[@class='article-container']/article").xpath("//section[@id='bodymatter']/div/div/text()").get();
+
+        String publishTime = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='core-self-citation']").xpath("//div[@class='core-date-published']/span/text()").get();
+        Date publishTimeDateTime = null;
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        List<Selectable> authorNodes = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='contributors']/span/span/span").nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/span/text()").get()).append(" ");
+        }
+
+
+        JSONArray authorEmail = new JSONArray();
+        List<Selectable> authorEmailSelectables = html.xpath("//div[@class='article-container']/article/header/div").xpath("//div[@class='contributors']").xpath("//span[@class='authors']").xpath("//span[@role='list']").xpath("//span[@property='author']").nodes();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String givenName = authorEmailSelectable.xpath("//span[@property='givenName']/text()").get();
+            String familyName = authorEmailSelectable.xpath("//span[@property='familyName']/text()").get();
+            String email = authorEmailSelectable.xpath("//a[@property='email']/text()").get();
+
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", givenName + "" + familyName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.SCIENCE.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(null)
+                .referenceInfo(null).build());
+    }
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    public static void main(String[] args) {
+        // 创建一个Spider，并把我们的处理器放进去
+        Spider.create(new Science4JournalArticlePageProcessor())
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.science.org/journal/science/insights?startPage=0")
+                .addPipeline(new ArticlePipeline())
+                // 开启5个线程执行，并开始爬取
+                .thread(5).run();
+    }
+}
 \ No newline at end of file
+package com.canrd.webmagic.processor;
+
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.KeywordUtil;
+import com.canrd.webmagic.processor.config.Agent;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import javax.annotation.Resource;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/1 14:19
+ * @version: 1.0
+ */
+@Slf4j
+@Component
+public class Science4JournalSearchPageProcessor implements PageProcessor {
+
+    @Resource
+    private Science4JournalArticlePageProcessor science4JournalArticlePageProcessor;
+
+    @Resource
+    private SeleniumDownloader seleniumDownloader;
+
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    /**
+     * 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
+     */
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setUserAgent(Agent.getRandom());
+
+    /**
+     * 定制爬虫逻辑的核心接口，在这里编写抽取逻辑
+     *
+     * @param page
+     */
+    @Override
+    public void process(Page page) {
+        doArticleList(page);
+    }
+
+    /**
+     * @param page
+     */
+    private void doArticleList(Page page) {
+        String url = page.getUrl().get();
+        /**
+         * 通过page.getHtml()可以获取到main函数中Spider.create(new BaiduHotSearchPageProcessor()).addUrl中的地址的网页内容
+         * 1、通过$或css()方法获取到该page html下某元素dom
+         */
+        Selectable selectable = page.getHtml().xpath("//div[@class=' search-result__body titles-results ']").select(new XpathSelector("div[@class='card pb-3 mb-4 border-bottom']"));
+        List<Selectable> nodes = selectable.nodes();
+
+        /**
+         * 获取到指定的dom后，从这些dom中提取元素内容。
+         */
+        for (int i = 0; i <= nodes.size() - 1; i++) {
+            String title = nodes.get(i).xpath("//div[@class='card pb-3 mb-4 border-bottom']/div").xpath("//div[@class='d-flex justify-content-between align-items-end']/div/span/h2/a/text()").get();
+            String time = nodes.get(i).xpath("//div[@class='card-meta align-middle mb-2 text-uppercase text-darker-gray']/span").nodes().get(2).xpath("//time/text()").get();
+            String link = nodes.get(0).links().get();
+            if (!KeywordUtil.containKeywordsInTitle(title)) {
+                SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+                try {
+                    Date publishTimeDateTime = formatter.parse(time);
+                    if (!publishTimeDateTime.before(DateUtil.localDate2Date(DateUtil.parseDate("2000-01-01", DateUtil.DATE)))) {
+//                        page.addTargetRequest(link);
+                        Spider.create(science4JournalArticlePageProcessor)
+                                .addUrl(link)
+                                .addPipeline(articlePipeline)
+                                .setDownloader(seleniumDownloader)
+                                // 开启5个线程执行，并开始爬取
+                                .thread(1).run();
+                        log.info("关键字文章列表链接：{},标题:{},文章链接：{}", url, title, link);
+                    }
+                } catch (ParseException e) {
+                    e.printStackTrace();
+                }
+
+            }
+        }
+
+    }
+
+    @Override
+    public Site getSite() {
+        return site;
+    }
+
+    public static void main(String[] args) {
+        // 创建一个Spider，并把我们的处理器放进去
+        Spider.create(new Science4JournalSearchPageProcessor())
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.science.org/journal/science/insights?startPage=0")
+                .addPipeline(new ArticlePipeline())
+                // 开启5个线程执行，并开始爬取
+                .thread(5).run();
+    }
+}
 \ No newline at end of file
-package com.canrd.webmagic.processor.config;
+package com.canrd.webmagic.processor.download;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+package com.canrd.webmagic.processor.download;
+
+import lombok.extern.slf4j.Slf4j;
+import org.openqa.selenium.By;
+import org.openqa.selenium.Cookie;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.downloader.AbstractDownloader;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.PlainText;
+
+import javax.annotation.Resource;
+import java.util.Map;
+
+/**
+ * @author: xms
+ * @description: TODO
+ * @date: 2024/4/26 16:36
+ * @version: 1.0
+ */
+@Slf4j
+@Component
+public class SeleniumDownloader extends AbstractDownloader {
+    private int sleepTime = 0;
+
+    @Resource
+    private WebDriver webDriver;
+
+    /**
+     * set sleep time to wait until load success
+     *
+     * @param sleepTime sleepTime
+     * @return this
+     */
+    public SeleniumDownloader setSleepTime(int sleepTime) {
+        this.sleepTime = sleepTime;
+        return this;
+    }
+
+    @Override
+    public Page download(Request request, Task task) {
+        Page page = Page.fail();
+        try {
+
+
+            log.info("downloading page " + request.getUrl());
+            webDriver.get(request.getUrl());
+            try {
+                if (sleepTime > 0) {
+                    Thread.sleep(sleepTime);
+                }
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            WebDriver.Options manage = webDriver.manage();
+            Site site = task.getSite();
+            if (site.getCookies() != null) {
+                for (Map.Entry<String, String> cookieEntry : site.getCookies()
+                        .entrySet()) {
+                    Cookie cookie = new Cookie(cookieEntry.getKey(),
+                            cookieEntry.getValue());
+                    manage.addCookie(cookie);
+                }
+            }
+
+            /*
+             * TODO You can add mouse event or other processes
+             *
+             * @author: bob.li.0718@gmail.com
+             */
+            try {
+                //休眠3秒就是为了动态的数据渲染完成后在进行获取
+                Thread.sleep(30000);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            WebElement webElement = webDriver.findElement(By.xpath("/html"));
+            String content = webElement.getAttribute("outerHTML");
+            page.setDownloadSuccess(true);
+            page.setRawText(content);
+            page.setHtml(new Html(content, request.getUrl()));
+            page.setUrl(new PlainText(request.getUrl()));
+            page.setRequest(request);
+            onSuccess(request, task);
+        } catch (Exception e) {
+            log.warn("download page {} error", request.getUrl(), e);
+            onError(request, task, e);
+        } finally {
+
+        }
+        return page;
+    }
+
+    @Override
+    public void setThread(int i) {
+
+    }
+}
@@ -14,7 +14,7 @@ import java.util.List;
 import java.util.Objects;
 @Component
-public class NatureArticlePipeline implements Pipeline {
+public class ArticlePipeline implements Pipeline {
     private ArticleService articleService;
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60
-Opera/8.0 (Windows NT 5.1; U; en)
 Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50
 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50
 Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0
-Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11
@@ -19,4 +17,6 @@ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)
 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36
-Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36
 \ No newline at end of file
+Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0
 \ No newline at end of file