mirror of
https://github.com/hs-web/hsweb-framework.git
synced 2026-05-07 21:55:57 +08:00
升级版本2.0-SNAPSHOT,修改项目结构
This commit is contained in:
zhouhao
2
.gitignore
vendored
2
.gitignore
vendored
@@ -18,4 +18,4 @@
|
||||
*.log
|
||||
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||
hs_err_pid*
|
||||
/hsweb-web-service-impl-common/data/
|
||||
/hsweb-web-service/hsweb-web-service-simple/data/
|
||||
|
||||
51
README.md
51
README.md
@@ -1,11 +1,17 @@
|
||||
## 后台管理基础框架 [](https://travis-ci.org/hs-web/hsweb-framework)
|
||||
基于maven,spring-boot,mybatis
|
||||
## 后台管理基础框架
|
||||
|
||||
[](https://travis-ci.org/hs-web/hsweb-framework)
|
||||
[](https://www.apache.org/licenses/LICENSE-2.0.html)
|
||||
|
||||
### 主要功能
|
||||
1. 基础dao,service,controller类，增删改查直接继承即可.
|
||||
2. 通用mybatis配置文件,支持多种条件查询自动生成,支持自动生成insert,update,delete语句 支持和查询相同的各种条件.
|
||||
3. 实现用户,权限管理;基于aop,注解,精确到按钮的权限控制.
|
||||
4. 动态表单功能,可在前端设计表单,动态生成数据库表,提供统一的增删改查接口.
|
||||
5. 数据库支持 mysql,oracle,h2
|
||||
5. 在线代码生成器,可自定义模板
|
||||
6. 动态多数据源,支持数据源热加载,热切换,支持分布式事务
|
||||
7. 数据库支持 mysql,oracle,h2
|
||||
8. websocket支持
|
||||
|
||||
### 其他组件
|
||||
1. [hsweb-easy-orm](https://github.com/hs-web/hsweb-easy-orm) :为动态表单设计的orm框架
|
||||
@@ -23,40 +29,13 @@
|
||||
$ mvn install -DskiptTests
|
||||
```
|
||||
|
||||
# 使用
|
||||
1. 开发文档
|
||||
2. FAQ
|
||||
# 此版本待完善功能
|
||||
1. 单元测试编写
|
||||
2. 增加定时调度,支持集群,任务采用脚本方式编写.
|
||||
3. 完善数据库持续集成,版本更新时自动更新数据库结构.
|
||||
4. 完善动态表单发布,表单发生变化后,自动重新发布(解决集群下,表单配置不一致).
|
||||
|
||||
# 演示
|
||||
1. 示例:[demo.hsweb.me](http://demo.hsweb.me)
|
||||
2. 测试用户:test (test2,test3,test4....) 密码:123456
|
||||
3. 演示项目源码:[hsweb-platform](https://github.com/hs-web/hsweb-platform)
|
||||
|
||||
# 许可
|
||||
[](https://www.apache.org/licenses/LICENSE-2.0.html)
|
||||
|
||||
# 加入hsweb
|
||||
<a target="_blank" href="http://shang.qq.com/wpa/qunwpa?idkey=3d66b5dd14991d7645af694e7649b373f3a9ce1216131094c78cb2348d542c41">
|
||||
<img border="0" src="http://pub.idqqimg.com/wpa/images/group.png" alt="hsweb.me" title="hsweb.me">515649185
|
||||
</a>
|
||||
|
||||
# 赞助 (支付宝)
|
||||
<img src="https://raw.githubusercontent.com/hs-web/hs-web.github.io/master/%E4%BB%98%E6%AC%BE%E7%BB%99%E6%88%91.png"
|
||||
width = "256" height = "256" alt="赞助" align=center />
|
||||
|
||||
# 鸣谢
|
||||
*按时间排序*
|
||||
|
||||
| 组织&个人 | 方式 |
|
||||
| ------------- |:-------------:|
|
||||
| [jetbrains.com](https://www.jetbrains.com) |正版 [IDE](https://www.jetbrains.com/Toolbox/) 授权 |
|
||||
| [@杭州-smart](https://github.com/JetBrainZP) |赞助: ￥150 (用于服务器升级) |
|
||||
| [@北京-50%](https://github.com/longfeizheng) |赞助: ￥50 (用于服务器升级) |
|
||||
| [@王乐](https://github.com/gmle) |赞助: ￥20 (用于服务器升级) |
|
||||
| 匿名 |赞助: ￥100 (用于服务器升级) |
|
||||
| [@西安-un](https://github.com/lw5826618) |赞助: ￥50 (用于服务器升级) |
|
||||
| [@重庆-下下](https://github.com/web-xiaxia) |赞助: ￥250 (用于服务器升级) |
|
||||
| [@天津-Mr.Chang](https://github.com/changdaye) |赞助: ￥100 (用于服务器升级) |
|
||||
| [@北京-青春无罪](https://github.com/gxz04220427) |赞助: ￥100 (用于服务器升级) |
|
||||
| [@kitkat](https://github.com/wy353208214) |赞助: ￥50 |
|
||||
|
||||
3. 演示项目源码:[hsweb-platform](https://github.com/hs-web/hsweb-platform)
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-bean</artifactId>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-concurrent</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-concurrent-cache</artifactId>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-concurrent</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-concurrent-lock</artifactId>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-controller</artifactId>
|
||||
@@ -23,7 +23,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-service-interface</artifactId>
|
||||
<artifactId>hsweb-web-service-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-core</artifactId>
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>hsweb-web-crawler</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<version>2.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.solr</groupId>
|
||||
<artifactId>solr-solrj</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>us.codecraft</groupId>
|
||||
<artifactId>webmagic-core</artifactId>
|
||||
<version>0.5.3</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>us.codecraft</groupId>
|
||||
<artifactId>webmagic-extension</artifactId>
|
||||
<version>0.5.3</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -1,70 +0,0 @@
|
||||
package org.hsweb.web.crawler;
|
||||
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-5-4.
|
||||
*/
|
||||
public class CrawlerResult {
|
||||
private String domain;
|
||||
|
||||
private String url;
|
||||
|
||||
private String content;
|
||||
|
||||
private String html;
|
||||
|
||||
private Date crawlerTime;
|
||||
|
||||
public String getDomain() {
|
||||
return domain;
|
||||
}
|
||||
|
||||
public void setDomain(String domain) {
|
||||
this.domain = domain;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public String getHtml() {
|
||||
return html;
|
||||
}
|
||||
|
||||
public void setHtml(String html) {
|
||||
this.html = html;
|
||||
}
|
||||
|
||||
public Date getCrawlerTime() {
|
||||
return crawlerTime;
|
||||
}
|
||||
|
||||
public void setCrawlerTime(Date crawlerTime) {
|
||||
this.crawlerTime = crawlerTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CrawlerResult{" +
|
||||
"domain='" + domain + '\'' +
|
||||
", url='" + url + '\'' +
|
||||
", content='" + content + '\'' +
|
||||
", html='" + html + '\'' +
|
||||
", crawlerTime=" + crawlerTime +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package org.hsweb.web.crawler.extracter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class DefaultHtmlContentExtractor implements HtmlContentExtractor {
|
||||
private static String clearLabel(String html) {
|
||||
html = html.replaceAll("(?is)<!DOCTYPE.*?>", "");
|
||||
html = html.replaceAll("(?is)<!--.*?-->", ""); // remove html comment
|
||||
html = html.replaceAll("(?is)<script.*?>.*?</script>", ""); // remove javascript
|
||||
html = html.replaceAll("(?is)<style.*?>.*?</style>", ""); // remove css
|
||||
html = html.replaceAll("&.{2,5};|&#.{2,5};", " "); // remove special char
|
||||
html = html.replaceAll("(?is)<.*?>", "");
|
||||
return html;
|
||||
}
|
||||
private int defaultThreshold=35;
|
||||
public DefaultHtmlContentExtractor(){}
|
||||
public DefaultHtmlContentExtractor(int threshold){
|
||||
this.defaultThreshold=threshold;
|
||||
}
|
||||
@Override
|
||||
public String parse(String html) {
|
||||
html = clearLabel(html);
|
||||
final int blocksWidth = 3;
|
||||
int start, end, threshold = defaultThreshold;
|
||||
StringBuilder text = new StringBuilder();
|
||||
List<Integer> indexDistribution = new ArrayList<>();
|
||||
List<String> lines = Arrays.asList(html.split("\n"));
|
||||
indexDistribution.clear();
|
||||
for (int i = 0; i < lines.size() - blocksWidth; i++) {
|
||||
int wordsNum = 0;
|
||||
for (int j = i; j < i + blocksWidth; j++) {
|
||||
lines.set(j, lines.get(j).replaceAll("\\s+", ""));
|
||||
wordsNum += lines.get(j).length();
|
||||
}
|
||||
indexDistribution.add(wordsNum);
|
||||
}
|
||||
start = -1;
|
||||
end = -1;
|
||||
boolean boolStart = false, boolEnd = false;
|
||||
text.setLength(0);
|
||||
for (int i = 0, len = indexDistribution.size(); i < len - 1; i++) {
|
||||
if (indexDistribution.get(i) > threshold && !boolStart) {
|
||||
if (indexDistribution.get(i + 1).intValue() != 0
|
||||
|| indexDistribution.get(i + 2).intValue() != 0
|
||||
|| indexDistribution.get(i + 3).intValue() != 0) {
|
||||
boolStart = true;
|
||||
start = i;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (boolStart) {
|
||||
if (indexDistribution.get(i).intValue() == 0
|
||||
|| indexDistribution.get(i + 1).intValue() == 0) {
|
||||
end = i;
|
||||
boolEnd = true;
|
||||
}
|
||||
}
|
||||
if (boolEnd) {
|
||||
StringBuilder tmp = new StringBuilder();
|
||||
for (int ii = start; ii <= end; ii++) {
|
||||
if (lines.get(ii).length() < 5) continue;
|
||||
tmp.append(lines.get(ii) + "\n");
|
||||
}
|
||||
String str = tmp.toString();
|
||||
if (str.toLowerCase().contains("copyright") || str.contains("版权所有")) continue;
|
||||
text.append(str);
|
||||
boolStart = boolEnd = false;
|
||||
}
|
||||
}
|
||||
return text.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
package org.hsweb.web.crawler.extracter;
|
||||
|
||||
/**
|
||||
* html正文提取器
|
||||
* Created by 浩 on 2015-09-07 0007.
|
||||
*/
|
||||
public interface HtmlContentExtractor {
|
||||
String parse(String html);
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package org.hsweb.web.crawler.extracter;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
|
||||
/**
|
||||
* Created by 浩 on 2015-09-07 0007.
|
||||
*/
|
||||
public class JsoupHtmlContentExtractor implements HtmlContentExtractor {
|
||||
|
||||
public JsoupHtmlContentExtractor(String select) {
|
||||
this.select = select;
|
||||
}
|
||||
|
||||
private String select;
|
||||
|
||||
@Override
|
||||
public String parse(String html) {
|
||||
Document document = Jsoup.parse(html);
|
||||
if (select == null)
|
||||
return document.text();
|
||||
return document.select(select).text();
|
||||
}
|
||||
|
||||
public String getSelect() {
|
||||
return select;
|
||||
}
|
||||
|
||||
public void setSelect(String select) {
|
||||
this.select = select;
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package org.hsweb.web.crawler.pipeline;
|
||||
|
||||
import org.hsweb.web.crawler.CrawlerResult;
|
||||
import org.hsweb.web.crawler.extracter.DefaultHtmlContentExtractor;
|
||||
import org.hsweb.web.crawler.extracter.HtmlContentExtractor;
|
||||
import org.hsweb.web.crawler.extracter.JsoupHtmlContentExtractor;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.ResultItems;
|
||||
import us.codecraft.webmagic.Task;
|
||||
import us.codecraft.webmagic.pipeline.Pipeline;
|
||||
import us.codecraft.webmagic.utils.UrlUtils;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-5-4.
|
||||
*/
|
||||
public abstract class AbstractPipeline implements Pipeline {
|
||||
|
||||
|
||||
public HtmlContentExtractor extractor;
|
||||
|
||||
public AbstractPipeline() {
|
||||
this(new DefaultHtmlContentExtractor(35));
|
||||
}
|
||||
|
||||
public AbstractPipeline(String selector) {
|
||||
this(new JsoupHtmlContentExtractor(selector));
|
||||
}
|
||||
|
||||
public AbstractPipeline(HtmlContentExtractor extractor) {
|
||||
this.extractor = extractor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ResultItems resultItems, Task task) {
|
||||
if (resultItems.isSkip()) return;
|
||||
Page page = resultItems.get("page");
|
||||
CrawlerResult result = new CrawlerResult();
|
||||
result.setCrawlerTime(new Date());
|
||||
result.setUrl(page.getUrl().get());
|
||||
result.setDomain(UrlUtils.getDomain(result.getUrl()));
|
||||
result.setHtml(page.getHtml().get());
|
||||
result.setContent(extractor.parse(result.getHtml()));
|
||||
process(result);
|
||||
}
|
||||
|
||||
public abstract void process(CrawlerResult result);
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package org.hsweb.web.crawler.pipeline;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.hsweb.web.crawler.CrawlerResult;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-5-4.
|
||||
*/
|
||||
public class SolrPipeline extends AbstractPipeline {
|
||||
private SolrServer solrServer;
|
||||
private String core = "hsweb-crawler";
|
||||
private Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
public SolrPipeline(String core) {
|
||||
this.solrServer = new HttpSolrServer("http://127.0.0.1:8983/solr/" + core);
|
||||
}
|
||||
|
||||
public SolrPipeline(SolrServer solrServer) {
|
||||
this.solrServer = solrServer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(CrawlerResult result) {
|
||||
try {
|
||||
logger.debug("save CrawlerResult " + result.getUrl());
|
||||
SolrInputDocument document = new SolrInputDocument();
|
||||
document.addField("url", result.getUrl());
|
||||
// document.addField("html", result.getHtml());
|
||||
document.addField("content_text_cn", result.getContent());
|
||||
document.addField("domain", result.getDomain());
|
||||
solrServer.add(document, 1000);
|
||||
} catch (Exception e) {
|
||||
logger.error("save CrawlerResult error!", e);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package org.hsweb.web.crawler.processor;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
import us.codecraft.webmagic.utils.UrlUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-5-4.
|
||||
*/
|
||||
public class SimplePageProcessor implements PageProcessor {
|
||||
private Site site;
|
||||
/**
|
||||
* 要抓取的页面
|
||||
*/
|
||||
private String crawlerUrlPattern;
|
||||
|
||||
/**
|
||||
* 要保存的页面
|
||||
*/
|
||||
private String saveUrlPattern;
|
||||
|
||||
|
||||
public SimplePageProcessor(String crawlerUrlPattern, String saveUrlPattern) {
|
||||
if (site == null)
|
||||
this.site = Site.me().setSleepTime(1000).setRetryTimes(5).setUseGzip(true);
|
||||
this.crawlerUrlPattern = "(" + crawlerUrlPattern.replace(".", "\\.").replace("*", "[^\"'#]*") + ")";
|
||||
this.saveUrlPattern = "(" + saveUrlPattern.replace(".", "\\.").replace("*", "[^\"'#]*") + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
List<String> requests = page.getHtml().links().regex(crawlerUrlPattern).all();
|
||||
page.addTargetRequests(requests);
|
||||
if (!page.getUrl().regex(saveUrlPattern).match())
|
||||
page.setSkip(true);
|
||||
page.putField("page", page);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return site;
|
||||
}
|
||||
|
||||
public void setSite(Site site) {
|
||||
this.site = site;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<!-- 控制台输出日志 -->
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} > %-5level %logger{35} - %m%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
<logger name="org.apache.http" level="ERROR" />
|
||||
<root level="DEBUG">
|
||||
<appender-ref ref="Console"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -1,51 +0,0 @@
|
||||
package org.hsweb.web.crawler;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.hsweb.web.crawler.pipeline.SolrPipeline;
|
||||
import org.hsweb.web.crawler.processor.SimplePageProcessor;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.scheduler.QueueScheduler;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-5-4.
|
||||
*/
|
||||
public class CrawlerTest {
|
||||
|
||||
public static void main(String[] args) throws SolrServerException {
|
||||
// Spider.create(new SimplePageProcessor("http://www.yiliu88.com*", "http://www.yiliu88.com/*.html"))
|
||||
// .addUrl("http://www.yiliu88.com")
|
||||
// .setScheduler(new QueueScheduler())
|
||||
// .addPipeline(new SolrPipeline("test"))
|
||||
// .thread(5)
|
||||
// .run();
|
||||
// HttpSolrServer server = new HttpSolrServer("http://127.0.0.1:8983/solr/test");
|
||||
// SolrQuery query = new SolrQuery();
|
||||
// query.setQuery("content_txt_en:hello zhangsan");
|
||||
// //mlt在查询时，打开/关闭 MoreLikeThisComponent 的布尔值
|
||||
// query.setParam("mlt", "true");
|
||||
// //fl 需要返回的字段
|
||||
// query.setParam("fl", "content_txt_en,id");
|
||||
// //mtl.fl 根据哪些字段判断相似度
|
||||
// query.setParam("mlt.fl", "content_txt_en");
|
||||
// //mlt.mintf 最小分词频率，在单个文档中出现频率小于这个值的词将不用于相似判断
|
||||
// query.setParam("mlt.mintf", "1");
|
||||
// //mlt.mindf 最小文档频率，所在文档的个数小于这个值的词将不用于相似判断
|
||||
// query.setParam("mlt.mindf", "1");
|
||||
// query.setParam("hl", "true");
|
||||
// query.setParam("hl.fl","content_txt_en");
|
||||
// query.setParam("hl.simple.pre","<span style='hl'>");
|
||||
// query.setParam("hl.simple.post","</span>");
|
||||
//
|
||||
// QueryResponse response = server.query(query);
|
||||
// response.getResponse().forEach(stringObjectEntry -> {
|
||||
// System.out.println(stringObjectEntry);
|
||||
// });
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<!-- 控制台输出日志 -->
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} > %-5level %logger{35} - %m%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
<logger name="org.apache.http" level="ERROR" />
|
||||
<root level="DEBUG">
|
||||
<appender-ref ref="Console"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -3,12 +3,12 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<artifactId>hsweb-web-dao</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-dao-interface</artifactId>
|
||||
<artifactId>hsweb-web-dao-api</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -10,7 +10,7 @@
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-dao-impl-mybatis</artifactId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<artifactId>hsweb-web-dao</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>hsweb-web-dao-impl-mybatis</artifactId>
|
||||
<artifactId>hsweb-web-dao-mybatis</artifactId>
|
||||
|
||||
<properties>
|
||||
<mybatis.version>3.3.2</mybatis.version>
|
||||
@@ -19,7 +19,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-dao-interface</artifactId>
|
||||
<artifactId>hsweb-web-dao-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -1,3 +1,3 @@
|
||||
# Auto Configure
|
||||
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
|
||||
# Auto Configure
|
||||
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
|
||||
org.hsweb.web.mybatis.MybatisDaoAutoConfiguration
|
||||
36
hsweb-web-dao/pom.xml
Normal file
36
hsweb-web-dao/pom.xml
Normal file
@@ -0,0 +1,36 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Copyright 2015-2016 http://hsweb.me
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<packaging>pom</packaging>
|
||||
<modules>
|
||||
<module>hsweb-web-dao-api</module>
|
||||
<module>hsweb-web-dao-mybatis</module>
|
||||
</modules>
|
||||
<artifactId>hsweb-web-dao</artifactId>
|
||||
|
||||
|
||||
|
||||
</project>
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-service-interface</artifactId>
|
||||
<artifactId>hsweb-web-service-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-oauth2</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-oauth2</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-service-interface</artifactId>
|
||||
<artifactId>hsweb-web-service-api</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-oauth2</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-dao-impl-mybatis</artifactId>
|
||||
<artifactId>hsweb-web-dao-mybatis</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -21,11 +21,11 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-oauth2</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>hsweb-web-oauth2-service-common</artifactId>
|
||||
<artifactId>hsweb-web-oauth2-service-simple</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -34,7 +34,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-service-impl-common</artifactId>
|
||||
<artifactId>hsweb-web-service-simple</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-web-oauth2</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<artifactId>hsweb-web-oauth2-service-common</artifactId>
|
||||
<artifactId>hsweb-web-oauth2-service-simple</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.hsweb</groupId>
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<parent>
|
||||
<artifactId>hsweb-framework</artifactId>
|
||||
<groupId>org.hsweb</groupId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
<modules>
|
||||
<module>hsweb-web-oauth2-mybatis</module>
|
||||
<module>hsweb-web-oauth2-controller</module>
|
||||
<module>hsweb-web-oauth2-service-common</module>
|
||||
<module>hsweb-web-oauth2-service-simple</module>
|
||||
<module>hsweb-web-oauth2-simple</module>
|
||||
<module>hsweb-web-oauth2-core</module>
|
||||
</modules>
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package org.hsweb.web.service.impl.system;
|
||||
|
||||
import org.hsweb.web.service.impl.AbstractTestCase;
|
||||
import org.hsweb.web.service.system.DataBaseManagerService;
|
||||
import org.junit.Test;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
/**
|
||||
* Created by zhouhao on 16-4-21.
|
||||
*/
|
||||
public class DataBaseManagerServiceImplTest extends AbstractTestCase {
|
||||
|
||||
@Resource
|
||||
private DataBaseManagerService dataBaseManagerService;
|
||||
|
||||
@Test
|
||||
public void testGetFieldList() throws Exception {
|
||||
dataBaseManagerService.getTableList();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
logging:
|
||||
config: classpath:logback.xml
|
||||
spring:
|
||||
aop:
|
||||
auto: true
|
||||
redis:
|
||||
host: 127.0.0.1
|
||||
port: 6389
|
||||
datasource:
|
||||
name: core
|
||||
url: jdbc:h2:mem:hsweb
|
||||
driverClassName: org.h2.Driver
|
||||
username: sa
|
||||
password:
|
||||
type: com.alibaba.druid.pool.DruidDataSource
|
||||
filters: stat
|
||||
maxActive: 20
|
||||
initialSize: 1
|
||||
maxWait: 60000
|
||||
minIdle: 1
|
||||
timeBetweenEvictionRunsMillis: 60000
|
||||
minEvictableIdleTimeMillis: 300000
|
||||
validationQuery: select 'x'
|
||||
testWhileIdle: true
|
||||
testOnBorrow: false
|
||||
testOnReturn: false
|
||||
poolPreparedStatements: true
|
||||
maxOpenPreparedStatements: 20
|
||||
mybatis:
|
||||
type-aliases-package: org.hsweb.web.bean.po
|
||||
mapper-locations: classpath*:org/hsweb/web/dao/impl/mybatis/mapper/oracle/**/*.xml
|
||||
config: classpath:mybatis-config.xml
|
||||
typeHandlers-package: org.hsweb.web.mybatis.handler
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user