!66 新增nebula数据源及脚本执行, 结果集解析为支持目前很多前端读取的可视化结构

Merge pull request !66 from JackieRiver/feature_nebula_module
This commit is contained in:
小东
2023-08-22 01:41:22 +00:00
committed by Gitee
17 changed files with 948 additions and 1 deletions

View File

@@ -0,0 +1,99 @@
---
title: nebula插件
date: 2023-08-16 09:16:55
---
### 引入依赖
```xml
<dependency>
<groupId>org.ssssssss</groupId>
<artifactId>magic-api-plugin-nebula</artifactId>
<version>magic-api-lastest-version</version>
</dependency>
```
### 配置
```yml
nebula:
hostAddress: ${NEBULA_HOSTADDRESS:localhost:9669}
userName: ${NEBULA_USERNAME:root}
password: ${NEBULA_PASSWORD:nebula}
```
### 使用
```js
import nebula;
var ngsl =
""""
USE db_name;MATCH p_=(p:`assignee`)-[*3]-(p2:`transferor`) where id(p2) == "阿里巴巴" or id(p)== "阿里巴巴" RETURN p_ limit 1000'
"""
var resultJson = nebula.executeJson(ngsl)
nebula.convert(resultJson)
nebula.executeNebulaModel(ngsl)
其他支持的方法不太常用, 这里不再一一列举, 可参考源码
org.ssssssss.magicapi.nebula.NebulaModule
```
#### 返回的数据格式为:
```
该结构的数据可被很多前端组件库支持进行可视化展示
```
如: [angv G6](http://antv-2018.alipay.com/zh-cn/g6/3.x/demo/index.html)
```json
{
"code": 0,
"message": "success",
"data": {
"nodes": [
{
"edgeSize": 1,
"assignee.name": "中航纽赫融资租赁(上海)有限公司",
"type": "vertex",
"assignee.addr": "上海市中国上海自由贸易试验区正定路530号A5库区集中辅助区三层318室",
"assignee.legal_person": "周勇",
"registrant.addr": "上海市浦东新区南泉路1261号",
"registrant.name": "中航国际租赁有限公司",
"id": "中航纽赫融资租赁(上海)有限公司",
"assignee.type": "企业"
},
{
"edgeSize": 15,
"type": "vertex",
"transferor.name": "陕西海富融资租赁有限公司",
"transferor.legal_person": "刘子瑜",
"transferor.type": "企业",
"transferor.addr": "陕西省西安市西安经济技术开发区未央路170号赛高城市广场2号楼企业总部大厦26层05单元",
"registrant.addr": "广东省深圳市前海深港合作区南山街道梦海大厦5035号前海华润金融中心T5写字楼1808",
"registrant.name": "深圳前海盈峰商业保理有限公司",
"id": "陕西海富融资租赁有限公司"
}, ...
],
"edges": [
{
"dst": "陕西海富融资租赁有限公司",
"src": "中航纽赫融资租赁(上海)有限公司",
"source": "中航纽赫融资租赁(上海)有限公司",
"label": "trans_with",
"type": "edge",
"target": "陕西海富融资租赁有限公司",
"name": "trans_with",
"ranking": 0,
"value": 0
},...
]
},
"timestamp": 1692149280167,
"requestTime": 1692149280143,
"executeTime": 24
}
```

View File

@@ -0,0 +1,33 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.ssssssss</groupId>
<artifactId>magic-api-plugins</artifactId>
<version>2.1.1</version>
</parent>
<artifactId>magic-api-plugin-nebula</artifactId>
<packaging>jar</packaging>
<name>magic-api-plugin-nebula</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<vesoft.version>3.5.0</vesoft.version>
</properties>
<dependencies>
<dependency>
<groupId>com.vesoft</groupId>
<artifactId>client</artifactId>
<version>${vesoft.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,104 @@
package org.ssssssss.magicapi.nebula;
import com.vesoft.nebula.client.graph.NebulaPoolConfig;
import com.vesoft.nebula.client.graph.data.HostAddress;
import com.vesoft.nebula.client.graph.net.NebulaPool;
import com.vesoft.nebula.client.graph.net.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.ssssssss.magicapi.core.config.MagicAPIProperties;
import org.ssssssss.magicapi.core.config.MagicPluginConfiguration;
import org.ssssssss.magicapi.core.model.Plugin;
import org.ssssssss.magicapi.utils.Assert;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* Nebula自动配置类
*/
@Configuration
@EnableConfigurationProperties(NebulaPoolProperties.class)
public class MagicNebulaConfiguration implements MagicPluginConfiguration {
private static final Logger logger = LoggerFactory.getLogger(MagicNebulaConfiguration.class);
private NebulaPoolProperties nebulaPoolProperties;
private final MagicAPIProperties properties;
public MagicNebulaConfiguration(MagicAPIProperties properties, NebulaPoolProperties nebulaPoolProperties) {
this.properties = properties;
this.nebulaPoolProperties = nebulaPoolProperties;
}
/**
* 创建nebula pool
* @param nebulaPoolProperties
* @return
*/
@Bean
public NebulaPool nebulaPool(@Autowired NebulaPoolProperties nebulaPoolProperties) {
Session session = null;
try {
NebulaPoolConfig nebulaPoolConfig = buildNebulaPoolConfig(nebulaPoolProperties);
Assert.isNotBlank(nebulaPoolProperties.getHostAddress(), "nebula.hostAddress 不能为空, 格式为 ip:port,ip:port 配置多个地址用逗号分隔");
String[] hostAddress = nebulaPoolProperties.getHostAddress().split(",");
List<HostAddress> addresses = Arrays.stream(hostAddress).map(address -> {
String[] ipAndPort = address.split(":");
Assert.isTrue(ipAndPort.length == 2, "nebula.hostAddress 格式错误, 格式为 ip:port,ip:port 配置多个地址用逗号分隔");
return new HostAddress(ipAndPort[0], Integer.parseInt(ipAndPort[1]));
}).collect(Collectors.toList());
NebulaPool pool = new NebulaPool();
pool.init(addresses, nebulaPoolConfig);
session = pool.getSession(nebulaPoolProperties.getUserName(), nebulaPoolProperties.getPassword(), nebulaPoolProperties.isReconnect());
return pool;
} catch (Exception e) {
logger.error("初始化nebula pool 异常", e);
throw new RuntimeException(e);
} finally {
logger.info("初始化nebula pool 完成");
Optional.ofNullable(session).ifPresent(Session::release);
}
}
/**
* 注入模块
* @return
*/
@Bean
public NebulaModule nebulaModule() {
return new NebulaModule();
}
@Override
public Plugin plugin() {
return new Plugin("Nebula");
}
public NebulaPoolConfig buildNebulaPoolConfig(NebulaPoolProperties nebulaPoolProperties) {
NebulaPoolConfig nebulaPoolConfig = new NebulaPoolConfig();
//将nebulaPoolProperties的同名属性赋值到nebulaPoolConfig
nebulaPoolConfig.setMinConnSize(nebulaPoolProperties.getMinConnsSize());
nebulaPoolConfig.setSslParam(nebulaPoolProperties.getSslParam());
nebulaPoolConfig.setWaitTime(nebulaPoolProperties.getWaitTime());
nebulaPoolConfig.setTimeout(nebulaPoolProperties.getTimeout());
nebulaPoolConfig.setMaxConnSize(nebulaPoolProperties.getMaxConnsSize());
nebulaPoolConfig.setIntervalIdle(nebulaPoolProperties.getIntervalIdle());
nebulaPoolConfig.setMinClusterHealthRate(nebulaPoolProperties.getMinClusterHealthRate());
nebulaPoolConfig.setIdleTime(nebulaPoolProperties.getIdleTime());
nebulaPoolConfig.setEnableSsl(nebulaPoolProperties.isEnableSsl());
return nebulaPoolConfig;
}
}

View File

@@ -0,0 +1,157 @@
package org.ssssssss.magicapi.nebula;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.vesoft.nebula.client.graph.data.ResultSet;
import com.vesoft.nebula.client.graph.net.NebulaPool;
import com.vesoft.nebula.client.graph.net.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.ssssssss.magicapi.core.annotation.MagicModule;
import org.ssssssss.magicapi.nebula.model.Edge;
import org.ssssssss.magicapi.nebula.model.NebulaModel;
import org.ssssssss.magicapi.nebula.model.Node;
import org.ssssssss.magicapi.nebula.response.*;
import org.ssssssss.script.annotation.Comment;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
@MagicModule("nebula")
public class NebulaModule {
@Autowired
private NebulaPool nebulaPool;
@Autowired
private NebulaPoolProperties nebulaPoolProperties;
private static final Logger logger = LoggerFactory.getLogger(NebulaModule.class);
/**
* 执行ngsl脚本, 返回json格式结果
*
* @param script
* @return
*/
@Comment("执行ngsl脚本, 返回json格式结果")
public Object executeJson(String script) {
Session session = getNebulaSession();
try {
String json = session.executeJson(script);
return json;
} catch (Exception e) {
logger.error("执行Nebula脚本异常, script: {}", script, e);
throw new RuntimeException(e);
} finally {
Optional.ofNullable(session).ifPresent(Session::release);
}
}
/**
* 执行ngsl脚本, 并解析为可视化格式
*
* @param script
* @return
*/
@Comment("执行ngsl脚本, 返回json格式结果, 并解析为可视化格式")
public NebulaModel executeNebulaModel(String script) {
Session session = getNebulaSession();
try {
String json = session.executeJson(script);
return convert(json);
} catch (Exception e) {
logger.error("执行Nebula脚本异常, script: {}", script, e);
throw new RuntimeException(e);
} finally {
Optional.ofNullable(session).ifPresent(Session::release);
}
}
/**
* 执行ngsl脚本, 返回ResultSet格式结果, 不可直接使用
*
* @param script
* @return
*/
@Comment("执行ngsl脚本, 返回ResultSet格式结果, 无法直接使用")
public Object execute(String script) {
Session session = getNebulaSession();
try {
ResultSet resultSet = session.execute(script);
return resultSet;
} catch (Exception e) {
logger.error("执行Nebula脚本异常, script: {}", script, e);
throw new RuntimeException(e);
} finally {
Optional.ofNullable(session).ifPresent(Session::release);
}
}
public Session getNebulaSession() {
try {
return nebulaPool.getSession(nebulaPoolProperties.getUserName(), nebulaPoolProperties.getPassword(), nebulaPoolProperties.isReconnect());
} catch (NoSuchBeanDefinitionException e) {
throw new RuntimeException(String.format("NebulaPool 未初始化, 或初始化异常, 请检查配置文件"));
} catch (Exception e) {
logger.error("获取nebula session 异常", e);
throw new RuntimeException(e);
}
}
@Comment("解析nebula结果为可视化格式")
public NebulaModel convert(String json) throws Exception {
ObjectMapper objectMapper = new ObjectMapper();
NebulaJsonBody response = objectMapper.readValue(json, NebulaJsonBody.class);
//状态码不为0则为异常, 解析提示异常信息
if (response.getErrorCode() != 0) {
logger.error("执行Nebula脚本异常, script: {}, errorMsg: {}", json, response.getErrorMsg());
throw new RuntimeException(response.getErrorMsg());
}
NebulaModel nebulaModel = new NebulaModel();
HashMap<String, Integer> nodeEdges = new HashMap<>();
List<NebulaJsonBody.Data> datas = response.getResults().get(0).getData();
for (int index = 0; index < datas.size(); index++) {
List<List<Element>> meta = datas.get(index).getMeta();
List<List<HashMap<String, Object>>> row = datas.get(index).getRow();
for (int i = 0; i < meta.get(0).size(); i++) {
Element element = meta.get(0).get(i);
HashMap<String, Object> elementDetail = row.get(0).get(i);
Node node = new Node();
Edge edge = new Edge();
if (element instanceof Vertex) {
node.setId(((Vertex) element).getId());
node.getProp().putAll(elementDetail);
nebulaModel.addNode(node);
} else if (element instanceof EdgeElement) {
edge.getProp().putAll(elementDetail);
EdgeId id = ((EdgeElement) element).getId();
edge.setTarget(id.getDst());
edge.setSource(id.getSrc());
edge.setLabel(id.getName());
edge.setValue(id.getRanking());
nebulaModel.getEdges().add(edge);
nodeEdges.put(id.getDst(), nodeEdges.getOrDefault(id.getDst(), 0) + 1);
nodeEdges.put(id.getSrc(), nodeEdges.getOrDefault(id.getSrc(), 0) + 1);
}
}
// 补充节点边的数量值
for (Node node : nebulaModel.getNodes()) {
node.setEdgeSize(nodeEdges.getOrDefault(node.getId(), 0));
}
}
return nebulaModel;
}
}

View File

@@ -0,0 +1,141 @@
package org.ssssssss.magicapi.nebula;
import com.vesoft.nebula.client.graph.data.SSLParam;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "nebula")
public class NebulaPoolProperties {
/** nebula 服务地址, 多个则逗号分割, 格式为 ip:port */
private String hostAddress;
/** nebula 用户名 */
private String userName;
/** nebula 密码 */
private String password;
private boolean reconnect = true;
/** nebula 连接池最小连接数 */
private int minConnsSize = 0;
/** nebula 连接池最大连接数 */
private int maxConnsSize = 10;
/** nebula 连接池最大等待时间 */
private int timeout = 0;
/** nebula 连接池空闲时间 */
private int idleTime = 0;
/** nebula 连接池心跳间隔 */
private int intervalIdle = -1;
private int waitTime = 0;
private double minClusterHealthRate = 1.0;
private boolean enableSsl = false;
private SSLParam sslParam = null;
public String getHostAddress() {
return hostAddress;
}
public void setHostAddress(String hostAddress) {
this.hostAddress = hostAddress;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public boolean isReconnect() {
return reconnect;
}
public void setReconnect(boolean reconnect) {
this.reconnect = reconnect;
}
public int getMinConnsSize() {
return minConnsSize;
}
public void setMinConnsSize(int minConnsSize) {
this.minConnsSize = minConnsSize;
}
public int getMaxConnsSize() {
return maxConnsSize;
}
public void setMaxConnsSize(int maxConnsSize) {
this.maxConnsSize = maxConnsSize;
}
public int getTimeout() {
return timeout;
}
public void setTimeout(int timeout) {
this.timeout = timeout;
}
public int getIdleTime() {
return idleTime;
}
public void setIdleTime(int idleTime) {
this.idleTime = idleTime;
}
public int getIntervalIdle() {
return intervalIdle;
}
public void setIntervalIdle(int intervalIdle) {
this.intervalIdle = intervalIdle;
}
public int getWaitTime() {
return waitTime;
}
public void setWaitTime(int waitTime) {
this.waitTime = waitTime;
}
public double getMinClusterHealthRate() {
return minClusterHealthRate;
}
public void setMinClusterHealthRate(double minClusterHealthRate) {
this.minClusterHealthRate = minClusterHealthRate;
}
public boolean isEnableSsl() {
return enableSsl;
}
public void setEnableSsl(boolean enableSsl) {
this.enableSsl = enableSsl;
}
public SSLParam getSslParam() {
return sslParam;
}
public void setSslParam(SSLParam sslParam) {
this.sslParam = sslParam;
}
}

View File

@@ -0,0 +1,69 @@
package org.ssssssss.magicapi.nebula.model;
import java.util.HashMap;
/**
* 描述node的方向的边
*/
public class Edge {
/**
* 起始节点的id
*/
private String source;
/**
* 终止节点的id
*/
private String target;
/**
* 边描述
*/
private String label;
private String value;
private HashMap<String, Object> prop = new HashMap<>();
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getTarget() {
return target;
}
public void setTarget(String target) {
this.target = target;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public HashMap<String, Object> getProp() {
return prop;
}
public void setProp(HashMap<String, Object> prop) {
this.prop = prop;
}
}

View File

@@ -0,0 +1,69 @@
package org.ssssssss.magicapi.nebula.model;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.ssssssss.script.annotation.Comment;
import java.util.*;
/**
* 经过加工后的nebula数据结构, 用于前端数据展示
* 目前很多前端组件库支持这种数据, 并可视化展示, 如ntV G6等
* @see <a href="@link:http://antv-2018.alipay.com/zh-cn/g6/3.x/demo/index.html">AntV G6</a>
*/
public class NebulaModel {
@JsonIgnore
private List<String> nodeIds = new ArrayList<>();
/**
* 包含的节点集合
*/
@Comment("包含的节点集合")
private List<Node> nodes = new ArrayList<>();
/**
* 包含的边集合
*/
@Comment("包含的边集合")
private List<Edge> edges = new ArrayList<>();
public List<String> getNodeIds() {
return nodeIds;
}
public void setNodeIds(List<String> nodeIds) {
this.nodeIds = nodeIds;
}
public List<Node> getNodes() {
return nodes;
}
public void setNodes(List<Node> nodes) {
this.nodes = nodes;
}
public List<Edge> getEdges() {
return edges;
}
public void setEdges(List<Edge> edges) {
this.edges = edges;
}
/**
* 添加节点, 根据id去重
* @param node
*/
@Comment("添加节点, 根据id去重")
public void addNode(Node node) {
String nodeId = Objects.toString(node.getId(), null);
if (nodeIds.contains(nodeId)) {
return;
}
nodeIds.add(nodeId);
nodes.add(node);
}
}

View File

@@ -0,0 +1,36 @@
package org.ssssssss.magicapi.nebula.model;
import java.util.HashMap;
public class Node {
private String id;
private int EdgeSize;
private HashMap<String, Object> prop = new HashMap<>();
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getEdgeSize() {
return EdgeSize;
}
public void setEdgeSize(int edgeSize) {
EdgeSize = edgeSize;
}
public HashMap<String, Object> getProp() {
return prop;
}
public void setProp(HashMap<String, Object> prop) {
this.prop = prop;
}
}

View File

@@ -0,0 +1,14 @@
package org.ssssssss.magicapi.nebula.response;
public class EdgeElement extends Element {
private EdgeId id;
public EdgeId getId() {
return id;
}
public void setId(EdgeId id) {
this.id = id;
}
}

View File

@@ -0,0 +1,50 @@
package org.ssssssss.magicapi.nebula.response;
public class EdgeId {
private String ranking;
private String name;
private Integer type;
private String dst;
private String src;
public String getRanking() {
return ranking;
}
public void setRanking(String ranking) {
this.ranking = ranking;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getType() {
return type;
}
public void setType(Integer type) {
this.type = type;
}
public String getDst() {
return dst;
}
public void setDst(String dst) {
this.dst = dst;
}
public String getSrc() {
return src;
}
public void setSrc(String src) {
this.src = src;
}
}

View File

@@ -0,0 +1,26 @@
package org.ssssssss.magicapi.nebula.response;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
property = "type")
@JsonSubTypes(value = {
@JsonSubTypes.Type(value = EdgeElement.class, name = "edge"),
@JsonSubTypes.Type(value = Vertex.class, name = "vertex")
})
public abstract class Element {
protected String type;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
}

View File

@@ -0,0 +1,132 @@
package org.ssssssss.magicapi.nebula.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.HashMap;
import java.util.List;
public class NebulaJsonBody {
private List<NebulaError> errors;
private List<Result> results;
public List<NebulaError> getErrors() {
return errors;
}
public void setErrors(List<NebulaError> errors) {
this.errors = errors;
}
public int getErrorCode() {
return this.errors.get(0).getCode();
}
public String getErrorMsg() {
return this.errors.get(0).getMessage();
}
public List<Result> getResults() {
return results;
}
public void setResults(List<Result> results) {
this.results = results;
}
public static class NebulaError {
private int code;
private String message;
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
}
public static class Result {
@JsonProperty("spaceName")
private String spaceName;
private List<Data> data;
private List<String> columns;
private NebulaError errors;
@JsonProperty("latencyInUs")
private long latencyInUs;
public String getSpaceName() {
return spaceName;
}
public void setSpaceName(String spaceName) {
this.spaceName = spaceName;
}
public List<Data> getData() {
return data;
}
public void setData(List<Data> data) {
this.data = data;
}
public List<String> getColumns() {
return columns;
}
public void setColumns(List<String> columns) {
this.columns = columns;
}
public NebulaError getErrors() {
return errors;
}
public void setErrors(NebulaError errors) {
this.errors = errors;
}
public long getLatencyInUs() {
return latencyInUs;
}
public void setLatencyInUs(long latencyInUs) {
this.latencyInUs = latencyInUs;
}
}
public static class Data {
private List<List<Element>> meta;
private List<List<HashMap<String, Object>>> row;
public List<List<Element>> getMeta() {
return meta;
}
public void setMeta(List<List<Element>> meta) {
this.meta = meta;
}
public List<List<HashMap<String, Object>>> getRow() {
return row;
}
public void setRow(List<List<HashMap<String, Object>>> row) {
this.row = row;
}
}
}

View File

@@ -0,0 +1,14 @@
package org.ssssssss.magicapi.nebula.response;
public class Vertex extends Element {
private String id;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}

View File

@@ -0,0 +1 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.ssssssss.magicapi.nebula.MagicNebulaConfiguration

View File

@@ -0,0 +1 @@
org.ssssssss.magicapi.nebula.MagicNebulaConfiguration

View File

@@ -23,6 +23,7 @@
<module>magic-api-plugin-elasticsearch</module>
<module>magic-api-plugin-cluster</module>
<module>magic-api-plugin-git</module>
<module>magic-api-plugin-nebula</module>
</modules>
<dependencies>
<dependency>

View File

@@ -33,7 +33,7 @@
<magic-script.version>1.8.8</magic-script.version>
<commons-compress.version>1.21</commons-compress.version>
<commons-io.version>2.7</commons-io.version>
<commons-text.version>1.6</commons-text.version>
<commons-text.version>1.10.0</commons-text.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<jakarta.version>6.0.0</jakarta.version>
<fastjson.version>1.2.83</fastjson.version>