提交

凌世锦
1 parent 1680e181
Showing 52 changed files with 1561 additions and 29 deletions
.idea/inspectionProfiles/Project_Default.xml
pom.xml
src/main/java/com/canrd/webmagic/DNS/DnsResolver.java
src/main/java/com/canrd/webmagic/controller/MatterController.java
src/main/java/com/canrd/webmagic/controller/NatureCommunicatiosController.java
src/main/java/com/canrd/webmagic/controller/NatureComputationalController.java
src/main/java/com/canrd/webmagic/controller/NatureEnergyController.java
src/main/java/com/canrd/webmagic/controller/NatureMaterialController.java
src/main/java/com/canrd/webmagic/controller/NatureMethodsController.java
src/main/java/com/canrd/webmagic/controller/NatureNanotechnologyController.java
src/main/java/com/canrd/webmagic/controller/NaturePhysicsController.java
src/main/java/com/canrd/webmagic/domain/ArticleTypeEnum.java
src/main/java/com/canrd/webmagic/processor/AdvancedEnergyMaterialPcoessor.java
src/main/java/com/canrd/webmagic/processor/ChemicalPagePcoessor.java
src/main/java/com/canrd/webmagic/processor/MatterPragePcoessor.java → src/main/java/com/canrd/webmagic/processor/MatterPagePcoessor.java
src/main/java/com/canrd/webmagic/processor/NatureCommunicatiosPcoessor.java
src/main/java/com/canrd/webmagic/processor/NatureComputationalPcoessor.java
src/main/java/com/canrd/webmagic/processor/NatureEnergyPagePcoessor.java
src/main/java/com/canrd/webmagic/processor/NatureMaterialPagePcoessor.java
src/main/java/com/canrd/webmagic/processor/NatureMethodsPcoessor.java
@@ -2,5 +2,6 @@
   <profile version="1.0">
     <option name="myName" value="Project Default" />
     <inspection_tool class="NonSerializableWithSerialVersionUIDField" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="SpringJavaInjectionPointsAutowiringInspection" enabled="false" level="ERROR" enabled_by_default="false" />
   </profile>
 </component>
 \ No newline at end of file
@@ -70,6 +70,12 @@
 <!--            <version>${browsermob.version}</version>-->
 <!--        </dependency>-->
+<!--        DNS-->
+        <dependency>
+            <groupId>dnsjava</groupId>
+            <artifactId>dnsjava</artifactId>
+            <version>2.1.8</version>
+        </dependency>
         <!-- webmagic核心库 -->
         <dependency>
+package com.canrd.webmagic.DNS;
+
+import org.xbill.DNS.*;
+
+public class DnsResolver {
+    public static String resolve(String domain) {
+        try {
+            Record[] records = new Lookup(domain, Type.A).run();
+            if (records != null && records.length > 0) {
+                ARecord aRecord = (ARecord) records[0];
+                return aRecord.getAddress().getHostAddress();
+            }
+        } catch (TextParseException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+}
 \ No newline at end of file
 package com.canrd.webmagic.controller;
 import com.canrd.webmagic.common.constant.ServerResult;
-import com.canrd.webmagic.processor.MatterPragePcoessor;
+import com.canrd.webmagic.processor.MatterPagePcoessor;
 import com.canrd.webmagic.processor.download.SeleniumDownloader;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -18,7 +18,7 @@ import javax.annotation.Resource;
 @Api("Matter")
 public class MatterController {
     @Resource
-    private MatterPragePcoessor matterPragePcoessor;
+    private MatterPagePcoessor matterPragePcoessor;
     @Resource
     private SeleniumDownloader seleniumDownloader;
@@ -26,11 +26,11 @@ public class MatterController {
     @GetMapping("/start")
     @ApiOperation("start")
     public ServerResult start()  {
-        Spider.create(new MatterPragePcoessor())
+        Spider.create(matterPragePcoessor)
                 // 添加这个Spider要爬取的网页地址
-                .addUrl("https://www.cell.com/matter/fulltext/S2590-2385(21)00631-7#%20")
+                .addUrl("https://www.cell.com/matter/home")
                 .setUUID(UuidUtil.getTimeBasedUuid().toString())
-                .setDownloader(seleniumDownloader.setSleepTime(30000))
+                .setDownloader(seleniumDownloader)
                 // 开启5个线程执行，并开始爬取
                 .thread(5).run();
         return ServerResult.success();
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureCommunicatiosPcoessor;
+import com.canrd.webmagic.processor.NatureEnergyPagePcoessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+@RestController
+@RequestMapping("/nature-communicatios/article")
+@Api("Nature")
+public class NatureCommunicatiosController {
+    @Resource
+    private NatureCommunicatiosPcoessor natureCommunicatiosPcoessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureCommunicatiosPcoessor)
+                .addUrl("https://www.nature.com/ncomms/articles?type=editorial")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+//                .setScheduler(new RedisScheduler("127.0.0.1"))
+                .thread(20).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureComputationalPcoessor;
+import com.canrd.webmagic.processor.NatureEnergyPagePcoessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/nature-computational/article")
+@Api("Nature")
+public class NatureComputationalController {
+    @Resource
+    private NatureComputationalPcoessor natureComputationalPcoessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureComputationalPcoessor)
+                .addUrl("https://www.nature.com/natcomputsci")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+                .setScheduler(new RedisScheduler("127.0.0.1"))
+                .thread(20).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureEnergyPagePcoessor;
+import com.canrd.webmagic.processor.NatureMaterialPagePcoessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/nature-energy/article")
+@Api("Nature")
+public class NatureEnergyController {
+    @Resource
+    private NatureEnergyPagePcoessor natureEnergyPagePcoessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureEnergyPagePcoessor)
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+                .setScheduler(new RedisScheduler("127.0.0.1"))
+                .thread(20).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.MatterPagePcoessor;
+import com.canrd.webmagic.processor.NatureMaterialPagePcoessor;
+import com.canrd.webmagic.processor.download.SeleniumDownloader;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/nature-material/article")
+@Api("Nature")
+public class NatureMaterialController {
+    @Resource
+    private NatureMaterialPagePcoessor natureMaterialPagePcoessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureMaterialPagePcoessor)
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.nature.com/nmat/articles")
+                .addUrl("https://www.nature.com/search?q=battery&journal=nmat&order=relevance")
+                .addUrl("https://www.nature.com/search?q=batteries&journal=nmat&order=relevance")
+                .addUrl("https://www.nature.com/search?q=battery")
+                .addUrl("https://www.nature.com/nature/research-articles")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+                .setScheduler(new RedisScheduler("127.0.0.1"))
+                // 开启5个线程执行，并开始爬取
+                .thread(60).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureMethodsPcoessor;
+import com.canrd.webmagic.processor.NatureNanotechnologyProcessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/nature-methods/article")
+@Api("Nature")
+public class NatureMethodsController {
+    @Resource
+    private NatureMethodsPcoessor natureMethodsPcoessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureMethodsPcoessor)
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.nature.com/nmeth/")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+                .setScheduler(new RedisScheduler("127.0.0.1"))
+                // 开启5个线程执行，并开始爬取
+                .thread(20).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureMaterialPagePcoessor;
+import com.canrd.webmagic.processor.NatureNanotechnologyProcessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+
+@RestController
+@RequestMapping("/nature-nanotechnology/article")
+@Api("Nature")
+public class NatureNanotechnologyController {
+    @Resource
+    private NatureNanotechnologyProcessor natureNanotechnologyProcessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(natureNanotechnologyProcessor)
+                // 添加这个Spider要爬取的网页地址
+                .addUrl("https://www.nature.com/nnano/")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+                .setScheduler(new RedisScheduler("127.0.0.1"))
+                // 开启5个线程执行，并开始爬取
+                .thread(60).run();
+        return ServerResult.success();
+    }
+}
+package com.canrd.webmagic.controller;
+
+import com.canrd.webmagic.common.constant.ServerResult;
+import com.canrd.webmagic.processor.NatureEnergyPagePcoessor;
+import com.canrd.webmagic.processor.NaturePhysicsProcessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scheduler.RedisScheduler;
+
+import javax.annotation.Resource;
+@RestController
+@RequestMapping("/nature-physics/article")
+@Api("Nature")
+public class NaturePhysicsController {
+    @Resource
+    private NaturePhysicsProcessor naturePhysicsProcessor;
+    @Resource
+    private ArticlePipeline articlePipeline;
+
+    @GetMapping("/start")
+    @ApiOperation("start")
+    public ServerResult start() {
+        Spider.create(naturePhysicsProcessor)
+                .addUrl("https://www.nature.com/nphys/")
+                .setUUID(UuidUtil.getTimeBasedUuid().toString())
+                .addPipeline(articlePipeline)
+//                .setScheduler(new RedisScheduler("127.0.0.1"))
+                .thread(20).run();
+        return ServerResult.success();
+    }
+}
 \ No newline at end of file
@@ -18,6 +18,13 @@ public enum ArticleTypeEnum {
     SCIENCE("science", "science网址"),
     SCIENCE_SPJ("science-spj", "science网址-spj"),
     UNIVIE_PHYSNANO("univie-physnano", "univie网址-physnano"),
+    NATURE_MATERIAL("nuture-material","nuture网站-material"),
+    NATURE_NANOTECHNOLOGY("nature-nanotechnology","nuture网站-nanotechnology"),
+    NATURE_PHYSICS("nature-physics","nuture网站-physics"),
+    NATURE_ENERGY("nature-energy","nuture网站-energy"),
+    NATURE_COMMUNICATIONS("nature-communications","nuture网站-communications"),
+    NATURE_COMPUTATIONAL_SCIENCE("nature-computational|science","nuture网站-computational|science"),
+    NATURE_METHODS("nature-methods","nuture网站-methods"),
     ;
     private String type;
     private String desc;
+package com.canrd.webmagic.processor;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+public class AdvancedEnergyMaterialPcoessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    
+}
 package com.canrd.webmagic.processor;
+import com.canrd.webmagic.DNS.DnsResolver;
 import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import com.gargoylesoftware.htmlunit.AbstractPage;
 import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Spider;
 import us.codecraft.webmagic.processor.PageProcessor;
 import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.PlainText;
 public class ChemicalPagePcoessor implements PageProcessor {
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
+
+    private Request request;
+    String domain = new PlainText(request.getUrl()).regex("//(.*?)/").get();
+
+    String ip = DnsResolver.resolve(domain);
+    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setDomain(domain).addCookie("ip", ip);;
     @Override
     public void process(Page page) {
 package com.canrd.webmagic.processor;
+import com.canrd.webmagic.processor.config.Agent;
 import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
 import org.springframework.stereotype.Component;
+import lombok.extern.slf4j.Slf4j;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Spider;
 import us.codecraft.webmagic.processor.PageProcessor;
 import us.codecraft.webmagic.selector.Html;
+
 @Component
-public class MatterPragePcoessor implements PageProcessor {
-    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
+@Slf4j
+public class MatterPagePcoessor implements PageProcessor {
+
+    private Site site = Site.me()
+            .setRetryTimes(3)
+            .setSleepTime(5)
+            .setUserAgent(Agent.getRandom());
+
     @Override
     public void process(Page page) {
+        //首页
+        if (page.getUrl().get().equals("https://www.cell.com/matter/home")){
+
+        }
+        //搜索页
+        else if (page.getUrl().get().contains("https://www.cell.com/action/doSearch?")) {
+        }
+        //详情页
+        else if (page.getUrl().get().contains("https://www.cell.com/matter/fulltext/")) {
+            doArticleContent(page);
+        }
     }
     @Override
     public Site getSite() {
-        return PageProcessor.super.getSite();
+        return this.site;
     }
-    public void doArticleContent(Page page){
+    public void doArticleContent(Page page) {
         Html html = page.getHtml();
+        log.info(String.valueOf(html));
         String articleCode = page.getUrl().get();
 //        html.xpath()
     }
     public static void main(String[] args) {
         // 创建一个Spider，并把我们的处理器放进去
-        Spider.create(new MatterPragePcoessor())
+        Spider.create(new MatterPagePcoessor())
                 // 添加这个Spider要爬取的网页地址
-                .addUrl("https://www.cell.com/matter/fulltext/S2590-2385(21)00631-7#%20")
+                .addUrl("https://www.cell.com/matter/home")
                 .addPipeline(new ArticlePipeline())
                 // 开启5个线程执行，并开始爬取
-                .thread(5).run();
+                .thread(1).run();
     }
 }
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class NatureCommunicatiosPcoessor implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        if (page.getUrl().get().equals("https://www.nature.com/ncomms/articles?type=editorial")){
+            getIndex(page);
+        } else if (page.getUrl().get().contains("https://www.nature.com/ncomms/articles?searchType=journalSearch&sort=PubDate&type=editorial&page=")) {
+            everyPage(page);
+        }else if (page.getUrl().get().contains("https://www.nature.com/articles/s41467-022-29269-6")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    public void getIndex(Page page){
+        String maxIndex = page.getHtml().xpath("//div[@class='u-mb-48']/nav/ul/li[5]/a/text()").get();
+        log.info(maxIndex);
+        String trim = maxIndex.trim();
+        int number = Integer.parseInt(trim);
+        System.out.printf("", number);
+        for (int i = 1; i <= number; i++) {
+            log.info(String.valueOf(i));
+            page.addTargetRequest("https://www.nature.com/ncomms/articles?searchType=journalSearch&sort=PubDate&type=editorial&page="+i);
+        }
+    }
+
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 1; i <= all.size(); i++) {
+            log.info(String.valueOf(i));
+            page.addTargetRequest("https://www.nature.com"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_COMMUNICATIONS.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class NatureComputationalPcoessor implements PageProcessor{
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+         if (url.contains("https://www.nature.com/nenergy/research-articles?searchType=journalSearch&sort=PubDate&page=")){
+            everyPage(page);
+        } else if (url.contains("https://www.nature.com/")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+            page.addTargetRequest("https://www.nature.com/"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_COMPUTATIONAL_SCIENCE.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.MatterPagePcoessor;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class NatureEnergyPagePcoessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://www.nature.com/nenergy/research-articles")){
+            getIndex(page);
+        } else if (url.contains("https://www.nature.com/nenergy/research-articles?searchType=journalSearch&sort=PubDate&page=")){
+            everyPage(page);
+        } else if (url.contains("https://www.nature.com/")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    public void getIndex(Page page){
+        String maxIndex = page.getHtml().xpath("//div[@class='u-mb-48']/nav/ul/li[6]/a/text()").get();
+        log.info(maxIndex);
+        String trim = maxIndex.trim();
+        int number = Integer.parseInt(trim);
+        System.out.printf("", number);
+        for (int i = 0; i < number; i++) {
+            page.addTargetRequest("https://www.nature.com/nenergy/research-articles?searchType=journalSearch&sort=PubDate&page="+i);
+        }
+    }
+
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+            page.addTargetRequest("https://www.nature.com/"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_ENERGY.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class NatureMaterialPagePcoessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://www.nature.com/nmat/articles")){
+            getIndex(page);
+        } else if (url.contains("https://www.nature.com/nmat/articles?searchType=journalSearch&sort=PubDate&page=")){
+            everyPage(page);
+        } else if (url.equals("https://www.nature.com/search?q=battery")) {
+            getIndex(page);
+        } else if (url.contains("https://www.nature.com/search?q=battery&page=")) {
+            everyPage(page);
+        } else if (url.contains("https://www.nature.com/articles")){
+            doArticleContent(page);
+        } else if (url.equals("https://www.nature.com/nature/research-articles")) {
+            getIndex(page);
+        } else if (url.contains("https://www.nature.com/nature/research-articles?searchType=journalSearch&sort=PubDate&page=")) {
+            everyPage(page);
+        } else if (url.equals("https://www.nature.com/search?q=battery&journal=nmat&order=relevance")) {
+            getIndex(page);
+        }else if (url.equals("https://www.nature.com/search?q=batteries&journal=nmat&order=relevance")) {
+            getIndex(page);
+        }else if (url.contains("https://www.nature.com/search?q=battery&order=relevance&journal=nmat&page=")) {
+            everyPage(page);
+        }else if (url.contains("https://www.nature.com/search?q=batteries&journal=nmat&page=")) {
+            everyPage(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite().setRetryTimes(3).setSleepTime(100);
+    }
+
+    public void getIndex(Page page){
+        String maxIndex = page.getHtml().xpath("//div[@class='u-mb-48']/nav/ul/li[6]/a/text()").get();
+        log.info(maxIndex);
+        String trim = maxIndex.trim();
+        int number = Integer.parseInt(trim);
+        if (page.getUrl().get().equals("https://www.nature.com/nmat/articles")){
+            for (int i = 1; i <= number; i++) {
+                page.addTargetRequest("https://www.nature.com/nmat/articles?searchType=journalSearch&sort=PubDate&page="+i);
+            }
+        }
+        if (page.getUrl().get().equals("https://www.nature.com/search?q=battery")){
+            for (int i = 1; i <= number; i++) {
+                page.addTargetRequest("https://www.nature.com/search?q=battery&page="+i);
+            }
+        }
+        if (page.getUrl().get().equals("https://www.nature.com/nature/research-articles")){
+            for (int i = 1; i <= number; i++) {
+                page.addTargetRequest("https://www.nature.com/nature/research-articles?searchType=journalSearch&sort=PubDate&page="+i);
+            }
+        }
+        if (page.getUrl().get().equals("https://www.nature.com/search?q=battery&journal=nmat&order=relevance")){
+            for (int i = 1; i <= number; i++) {
+                page.addTargetRequest("https://www.nature.com/search?q=battery&order=relevance&journal=nmat&page="+i);
+            }
+        }
+        if (page.getUrl().get().equals("https://www.nature.com/search?q=batteries&journal=nmat&order=relevance")){
+            for (int i = 1; i <= number; i++) {
+                page.addTargetRequest("https://www.nature.com/search?q=batteries&journal=nmat&page="+i);
+            }
+        }
+    }
+
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+//            log.info(all.get(i));
+            page.addTargetRequest("https://www.nature.com"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        if (page.getUrl().get().contains("redirect") || !page.getUrl().get().contains("nature")) {
+            return;
+        }
+        //解析页面
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_MATERIAL.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nmat/articles?searchType=journalSearch&sort=PubDate&page=1")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class NatureMethodsPcoessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://www.nature.com/nmeth/")) {
+            everyPage(page);
+        }else if (url.contains("https://www.nature.com/")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+            page.addTargetRequest("https://www.nature.com/"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_METHODS.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class NatureNanotechnologyProcessor implements PageProcessor {
+
+    //目前只有一页
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://www.nature.com/nnano/")){
+            everyPage(page);
+        }  else if (url.contains("https://www.nature.com/")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+
+    public void getIndex(Page page){
+        String maxIndex = page.getHtml().xpath("//div[@class='u-mb-48']/nav/ul/li[6]/a/text()").get();
+        log.info(maxIndex);
+        String trim = maxIndex.trim();
+        int number = Integer.parseInt(trim);
+        System.out.printf("", number);
+        for (int i = 0; i < number; i++) {
+            page.addTargetRequest("https://www.nature.com/nenergy/research-articles?searchType=journalSearch&sort=PubDate&page="+i);
+        }
+    }
+
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+            page.addTargetRequest("https://www.nature.com/"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_NANOTECHNOLOGY.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+}
+package com.canrd.webmagic.processor;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.canrd.webmagic.common.utils.DateUtil;
+import com.canrd.webmagic.common.utils.StringUtils;
+import com.canrd.webmagic.domain.ArticleTypeEnum;
+import com.canrd.webmagic.domain.dto.ArticleDO;
+import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.selector.XpathSelector;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class NaturePhysicsProcessor implements PageProcessor{
+    @Override
+    public void process(Page page) {
+        String url = page.getUrl().get();
+        if (url.equals("https://www.nature.com/nphys/")) {
+            everyPage(page);
+        }else if (url.contains("https://www.nature.com/")){
+            doArticleContent(page);
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return PageProcessor.super.getSite();
+    }
+    public void everyPage(Page page){
+        List<String> all = page.getHtml().xpath("//div[@class='u-flex-direction-column']/h3/a//@href").all();
+        for (int i = 0; i < all.size(); i++) {
+            page.addTargetRequest("https://www.nature.com/"+all.get(i));
+        }
+    }
+
+    private void doArticleContent(Page page) {
+        Html html = page.getHtml();
+        String articleCode = page.getUrl().get();
+        Selectable headSelectable = html.xpath("//div[@class='c-article-header']/header");
+        List<Selectable> authorEmailSelectables = html.xpath("//p[@id='corresponding-author-list']/a").nodes();
+        Selectable referencesSelectable = html.xpath("//ol[@class='c-article-references']").select(new XpathSelector("li[@class='c-article-references__item js-c-reading-companion-references-item']"));
+        Selectable authorAddressSelectable = html.xpath("//ol[@class='c-article-author-affiliation__list']").select(new XpathSelector("li"));
+
+        String title = headSelectable.xpath("//div/h1/text()").get();
+        if (StringUtils.isBlank(title)) {
+            title = headSelectable.xpath("//h1/text()").get();
+        }
+        String articleDesc = html.xpath("//div[@class='c-article-section__content']/p/text()").get();
+        String publishTime;
+        Date publishTimeDateTime = null;
+        try {
+            publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(2).xpath("//li/time/text()").get();
+        } catch (Exception e) {
+            try {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(1).xpath("//li/time/text()").get();
+            } catch (Exception e1) {
+                publishTime = headSelectable.xpath("//ul").nodes().get(0).xpath("//li").nodes().get(0).xpath("//li/time/text()").get();
+            }
+        }
+        SimpleDateFormat formatter = new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH);
+
+        try {
+            publishTimeDateTime = formatter.parse(publishTime);
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        Selectable authorSelectable = headSelectable.xpath("//ul").nodes().get(1).select(new XpathSelector("li[@class='c-article-author-list__item']"));
+        List<Selectable> authorNodes = authorSelectable.nodes();
+        StringBuffer authorName = new StringBuffer();
+        for (Selectable node : authorNodes) {
+            authorName.append(node.xpath("//a/text()"));
+        }
+
+        JSONArray authorAddress = new JSONArray();
+        List<Selectable> authorAddressList = authorAddressSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(authorAddressList)) {
+            for (Selectable selectable : authorAddressList) {
+                String address = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__address']/text()").get();
+                String authorNames = selectable.xpath("//p").xpath("//p[@class='c-article-author-affiliation__authors-list']/text()").get();
+                JSONObject object = new JSONObject();
+                object.put("address", address);
+                object.put("authorNames", authorNames);
+                authorAddress.add(object);
+            }
+        }
+
+        JSONArray references = new JSONArray();
+        List<Selectable> referenceList = referencesSelectable.nodes();
+        if (CollectionUtils.isNotEmpty(referenceList)) {
+            for (Selectable reference : referenceList) {
+                String referenceTitle = reference.xpath("//p").xpath("//p[@class='c-article-references__text']/text()").get();
+                List<Selectable> referenceLinks = reference.xpath("//p").xpath("//p[@class='c-article-references__links u-hide-print']").links().nodes();
+                List<String> links = new ArrayList<>();
+                if (CollectionUtils.isNotEmpty(referenceLinks)) {
+                    links = referenceLinks.stream().map(x -> x.get()).collect(Collectors.toList());
+                }
+                JSONObject object = new JSONObject();
+                object.put("referenceTitle", referenceTitle);
+                object.put("links", links);
+//                if (CollectionUtils.isNotEmpty(links)) {
+//                    page.addTargetRequests(links.stream().filter(x -> x.contains("nature")).collect(Collectors.toList()));
+//                }
+                references.add(object);
+            }
+        }
+
+        JSONArray authorEmail = new JSONArray();
+        for (Selectable authorEmailSelectable : authorEmailSelectables) {
+            String[] split = authorEmailSelectable.xpath("//a").links().get().split(":");
+            String email = Objects.isNull(split) ? "" : split[split.length - 1];
+            String authorEmailName = authorEmailSelectable.xpath("//a/text()").get();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("authorEmailName", authorEmailName);
+            jsonObject.put("email", email);
+            authorEmail.add(jsonObject);
+        }
+        log.info("文章链接：{},发布时间:{},标题:{},作者:{},邮箱信息:{}", articleCode, publishTime, title, authorEmail.toJSONString());
+
+        page.putField("article", ArticleDO.builder()
+                .articleType(ArticleTypeEnum.NATURE_PHYSICS.getType())
+                .articleCode(articleCode)
+                .authorName(authorName.toString())
+                .title(title)
+                .publishTime(Objects.isNull(publishTimeDateTime) ? publishTime : DateUtil.format(publishTimeDateTime, DateUtil.DATE))
+                .emailInfo(authorEmail.toJSONString())
+                .articleDesc(articleDesc)
+                .authorAddress(authorAddress.toJSONString())
+                .referenceInfo(references.toJSONString()).build());
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new MatterPagePcoessor())
+                .addUrl("https://www.nature.com/nenergy/research-articles")
+                .addPipeline(new ArticlePipeline())
+                .thread(1).run();
+    }
+}
@@ -55,7 +55,6 @@ public class NatureSearchPageProcessor implements PageProcessor {
         }else {
             doArticleContent(page);
         }
-
     }
     /**
@@ -140,7 +140,6 @@ public class Science4JournalSearchPageProcessor implements PageProcessor {
             }
         }
-
     }
     @Override
@@ -3,10 +3,9 @@ package com.canrd.webmagic.processor.download;
 import com.canrd.webmagic.config.SeleniumConfig;
 import com.canrd.webmagic.processor.config.Agent;
 import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.By;
-import org.openqa.selenium.Cookie;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
+import org.openqa.selenium.*;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
 import org.springframework.stereotype.Component;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Request;
@@ -28,7 +27,7 @@ import java.util.Map;
 @Slf4j
 @Component
 public class SeleniumDownloader extends AbstractDownloader {
-    private int sleepTime = 3000;
+    private int sleepTime = 3000000;
     @Resource
     private SeleniumConfig config;
@@ -64,8 +63,23 @@ public class SeleniumDownloader extends AbstractDownloader {
             }
             log.info("downloading page " + request.getUrl());
-
             webDriver.get(request.getUrl());
+            if (request.getUrl().equals("https://www.cell.com/matter/home")) {
+                WebElement searchText = webDriver.findElement(By.id("searchText"));
+                searchText.sendKeys("Aluminum foil");
+                WebElement element = webDriver.findElement(By.xpath("//div[@class='quick-search__toggle']/button"));
+                element.submit();
+                WebDriverWait wait = new WebDriverWait(webDriver, 30);
+                wait.until(ExpectedConditions.urlContains("https://www.cell.com/action/doSearch?"));
+//                wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//div[@class='cb-c']")));
+//                WebElement cloudFlare = webDriver.findElement(By.xpath("//div[@class='cb-c']/label/input"));
+//                if (cloudFlare!=null){
+//                    cloudFlare.click();
+//                }
+            }
+            if (request.getUrl().contains("https://www.cell.com/action/doSearch?")){
+
+            }
             try {
                 if (sleepTime > 0) {
                     //休眠3秒就是为了动态的数据渲染完成后在进行获取
@@ -75,6 +89,18 @@ public class SeleniumDownloader extends AbstractDownloader {
                 e.printStackTrace();
             }
+//            WebElement targetElement;
+//            do {
+//                try {
+//                    targetElement = webDriver.findElement(By.xpath("//h2[@class=\"h2\"]"));
+//                    log.info(String.valueOf(targetElement));
+//                    log.info("等待验证中");
+//                    Thread.sleep(sleepTime); // 等待一段时间后再检查
+//                } catch (NoSuchElementException e) {
+//                    targetElement = null; // 如果找不到特定元素，则退出循环
+//                }
+//            } while (targetElement != null);
+
             WebElement webElement = webDriver.findElement(By.xpath("/html"));
             String content = webElement.getAttribute("outerHTML");
             page.setDownloadSuccess(true);
@@ -59,13 +59,14 @@ public class MyChromeDriver implements BrowserDriver{
         options.addArguments("blink-settings=imagesEnabled=false");
         // 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
         options.addArguments("--headless");
+        //进入指定地址
+//        options.setExperimentalOption("debuggerAddress", "127.0.0.1:9222");
         //禁用 blink 特征
         options.addArguments("disable-blink-features=AutomationControlled");
         options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
         options.setExperimentalOption("useAutomationExtension", false);
         options.addArguments("--remote-allow-origins=*");
-
         String os_name = System.getProperty("os.name");
         // 判断是否是windows系统
         if (os_name.toLowerCase().startsWith("win")) {
@@ -27,11 +27,11 @@ public class ArticlePipeline implements Pipeline {
     public void process(ResultItems resultItems, Task task) {
         ArticleDO articleDO = resultItems.get("article");
         if (Objects.nonNull(articleDO)) {
-            List<ArticleDO> natureArticleDO = articleService.list(new LambdaQueryWrapper<ArticleDO>().eq(ArticleDO::getArticleCode, articleDO.getArticleCode()));
-            if (CollectionUtils.isNotEmpty(natureArticleDO)) {
-                return;
-            }
-            articleService.save(articleDO);
+                List<ArticleDO> natureArticleDO = articleService.list(new LambdaQueryWrapper<ArticleDO>().eq(ArticleDO::getArticleCode, articleDO.getArticleCode()));
+                if (CollectionUtils.isNotEmpty(natureArticleDO)) {
+                    return;
+                }
+                articleService.save(articleDO);
         }
     }
 }
@@ -59,7 +59,7 @@ spring:
     testOnReturn: true
     password: 123456
     time-between-eviction-runs-millis: 1000
-    url: jdbc:mysql://localhost:3306/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
+    url: jdbc:mysql://localhost:3306/webpage?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root
   redis:
     database: 0
@@ -59,7 +59,7 @@ spring:
     testOnReturn: true
     password: 123456
     time-between-eviction-runs-millis: 1000
-    url: jdbc:mysql://localhost:3306/webmagic?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
+    url: jdbc:mysql://localhost:3306/webpage?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai&useSSL=false&autoReconnect=true&failOverReadOnly=false&maxReconnects=10&allowMultiQueries=true&useAffectedRows=true&autoReconnectForPools=true
     username: root
   redis:
     database: 0