凌世锦
authored
|
1
2
3
|
package com.canrd.webmagic.controller;
|
凌世锦
authored
|
4
5
6
7
8
9
|
import com.canrd.webmagic.common.constant.ServerResult;
import com.canrd.webmagic.processor.ChemicalPcoessor;
import com.canrd.webmagic.processor.ChemicalsciencePcoessor;
import com.canrd.webmagic.processor.MatterPagePcoessor;
import com.canrd.webmagic.processor.download.SeleniumDownloader;
import com.canrd.webmagic.processor.pipeline.ArticlePipeline;
|
凌世锦
authored
|
10
|
import io.swagger.annotations.Api;
|
凌世锦
authored
|
11
12
13
|
import io.swagger.annotations.ApiOperation;
import org.apache.logging.log4j.core.util.UuidUtil;
import org.springframework.web.bind.annotation.GetMapping;
|
凌世锦
authored
|
14
15
|
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
|
凌世锦
authored
|
16
17
18
19
20
|
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.RedisScheduler;
import javax.annotation.Resource;
|
凌世锦
authored
|
21
22
|
@RestController
|
凌世锦
authored
|
23
24
|
@RequestMapping("/chemical/article")
@Api("Chemical")
|
凌世锦
authored
|
25
|
public class ChemicalController {
|
凌世锦
authored
|
26
27
28
29
30
31
32
33
|
@Resource
private ChemicalPcoessor chemicalPcoessor;
@Resource
private SeleniumDownloader seleniumDownloader;
@Resource
private ArticlePipeline articlePipeline;
|
凌世锦
authored
|
34
|
|
凌世锦
authored
|
35
36
37
38
39
40
41
42
43
|
@GetMapping("/start")
@ApiOperation("start")
public ServerResult start() {
Spider.create(chemicalPcoessor)
.setUUID(UuidUtil.getTimeBasedUuid().toString())
.addRequest(new Request("https://pubs.rsc.org/en/results?searchtext=battery"))
// .setScheduler(new RedisScheduler("127.0.0.1"))
.addPipeline(articlePipeline)
// .setDownloader(seleniumDownloader)
|
凌世锦
authored
|
44
|
.thread(100).run();
|
凌世锦
authored
|
45
46
|
return ServerResult.success();
}
|
凌世锦
authored
|
47
|
}
|