Commit bb63f128 authored by ymwang's avatar ymwang

项目代码迁移

parent e015bf93
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**
!**/src/test/**
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
### VS Code ###
.vscode/
FROM centos
MAINTAINER Joliu<ljo0412@live.com>
RUN yum install -y http://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm &&\
yum install -y iperf iptables tc
RUN yum install -y net-tools.x86_64
RUN echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-security main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-updates main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-proposed main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-security main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-updates main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-proposed main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-backports main restricted universe multiverse
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-business-client</artifactId>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-commom-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-business-utils</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>cnf-client-starter</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<!--这里写上main方法所在类的路径-->
<configuration>
<mainClass>top.ninwoo.dbc.client.ClientStarter</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package top.ninwoo.dbc.client;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
//这个注解告诉我们,这是一个springboot项目
@SpringBootApplication
@EnableAsync
public class ClientStarter {
public static void main(String[] args) {
SpringApplication.run(ClientStarter.class,args);
}
}
package top.ninwoo.dbc.client.configure;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
@Configuration
@EnableAsync // 开启异步调用 多线程
public class AsyncTaskConfig{
@Bean("taskExecutor")
public Executor taskExecutor() {
// 新建一个任务执行器
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setCorePoolSize(10);////核心线程池大小
taskExecutor.setMaxPoolSize(30);// 设置最大的线程数量
taskExecutor.setQueueCapacity(25);// 等待队列
taskExecutor.initialize();// 如果不初始化,导致找不到执行器
return taskExecutor;
}
}
package top.ninwoo.dbc.client.configure;
import top.ninwoo.dbc.api.service.DistributedComService;
import top.ninwoo.dbc.utils.FileServiceImplement;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
@Configuration
public class ClientConfigure {
@Primary
@Bean(name="fileRestTemplates")
public RestTemplate restTemplate(ClientHttpRequestFactory factory){
return new RestTemplate(factory);
}
@Bean
public ClientHttpRequestFactory simpleClientHttpRequestFactory(){
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setReadTimeout(5000);//单位为ms
factory.setConnectTimeout(5000);//单位为ms
return factory;
}
@Bean
public DistributedComService distributedComService() {
return new FileServiceImplement();
}
}
package top.ninwoo.dbc.client.configure;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
@Configuration
public class StaticResourceConfiguration extends WebMvcConfigurerAdapter {
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
registry.addResourceHandler("/file/**").addResourceLocations("file:/tmp/static/");
//registry.addResourceHandler("/file/**").addResourceLocations("file:/F:/resources/static/");
super.addResourceHandlers(registry);
}
}
package top.ninwoo.dbc.client.controller;
import top.ninwoo.dbc.api.po.FileSlice;
import top.ninwoo.dbc.client.service.FileService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
@RestController
public class ClientController {
/* @RequestMapping("/helloClient")
@ResponseBody
public String helloClient(){
return "hello,client";
}*/
@Resource
private FileService fileService;
@GetMapping("test1")
public String sendData(String name) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileBytes(name.getBytes());
fileSlice.setFileId(1111L);
String result = fileService.sendFileSlice("127.0.0.1:8080", fileSlice);
return result;
}
/* @GetMapping("sendFile")
public String sendFile(String name, Long fileId) {
BufferedImage image = fileService.readImage(name);
return fileService.sendFile(fileId, image);
}*/
@GetMapping("getFile")
public String getFile(String name, Long fileId) {
return fileService.getFile(name, fileId);
}
}
package top.ninwoo.dbc.client.controller;
import top.ninwoo.dbc.client.service.FileService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.FileNotFoundException;
import java.io.IOException;
@Controller
public class FileController {
@Autowired
private FileService fileService;
@Value("${dbc.directory.output}")
private String directoryOutput;
/*文件上传*/
// 访问路径为:http://ip:port/upload
@RequestMapping(value = "/upload",method = RequestMethod.GET)
public String upload() {
return "fileUpload";
}
@RequestMapping(value = "/upload", method = RequestMethod.POST)
@ResponseBody
public String upload(@RequestParam("file") MultipartFile file, @RequestParam("fileId") Long fileId) {
if (!file.isEmpty()) {
try {
//BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(new File(file.getOriginalFilename())));
//FileOutputStream out = new FileOutputStream(new File(file.getOriginalFilename()));
BufferedImage image = ImageIO.read(file.getInputStream());
fileService.sendFile(fileId,image);
/*out.flush();
out.close();*/
} catch (FileNotFoundException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
} catch (IOException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
}
return "上传成功!";
} else {
return "上传失败,因为文件是空的.";
}
}
/*文件下载*/
//访问路径为:http://ip:port/download?name=x.jpg&fileId=1111
@RequestMapping("download")
public String view(String name, Long fileId, Model model) {
fileService.getFile(name, fileId);
model.addAttribute("fileName", name);
return "fileDownload";
}
}
package top.ninwoo.dbc.client.service;
import top.ninwoo.dbc.api.po.*;
import top.ninwoo.dbc.api.service.DistributedComService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.*;
//import top.ninwoo.bishe.starter.service.NetworkService;
/**
* 文件服务
*/
@Service
public class FileService{
@Value("${dbc.directory.output}")
private String directoryOutput;
@Qualifier(value="fileRestTemplates")
@Autowired
private RestTemplate restTemplate;
/*@Autowired
private NetworkService networkService;*/
@Autowired
private DistributedComService distributedComService;
/* private String[] ipList=null;
@PostConstruct
public void init() {
ipList = getIpList(11113l,"dbc_server");
System.out.println(Arrays.toString(ipList));
}*/
/*获取容器ip*/
String[] ipList = new String[]{"127.0.0.1:8080","127.0.0.1:8081","127.0.0.1:8082","127.0.0.1:8083","127.0.0.1:8084",
"127.0.0.1:8085","127.0.0.1:8086"};
/* public String[] getIpList(Long clusterId, String appName) {
List<String> ipListS = networkService.getIpListByAppName(clusterId, appName);//用于存储容器ip的list集合
String[] containerIp = new String[ipListS.size()];//将集合中的元素存在数组里,因为下边用的是数组
for (int i = 0; i < ipListS.size(); i++) {
if(!ipListS.isEmpty()){
String ip_tmp = ipListS.get(i);
String[] split_list = ip_tmp.split("/");
containerIp[i] = split_list[0]+":8082";
}
}
return containerIp;
}*/
/**
* 发送文件到分布式节点
* @param fileId
* @param bufferedImage
* @return
*/
@Async("taskExecutor")
public String sendFile(Long fileId, BufferedImage bufferedImage) {
// 通过集群服务器接口获取当前集群的节点数量
int sliceNum = ipList.length;
/*ipList = getIpList(11113l,"dbc_server");
int sliceNum = ipList.length;*/
// todo 这里需要制定文件的fileId
SplitResult splitResult = distributedComService.fileSplit(bufferedImage, sliceNum);
int ipIndex = 0;
for (FileSlice slice : splitResult.getFileSliceList()) {
slice.setFileId(fileId);
//异常处理
while(true){
try {
sendFileSlice(ipList[ipIndex], slice);
ipIndex = (ipIndex + 1) % ipList.length;
break;
}catch (Exception e){
ipIndex = (ipIndex + 1) % ipList.length;
}
}
}
return "success";
/*for (FileSlice slice : splitResult.getFileSliceList()) {
slice.setFileId(fileId);
String result = sendFileSlice(ipList[ipIndex], slice);
ipIndex = (ipIndex + 1) % ipList.length;
if(result.equals("failed")) {
return "fail";
}
}
return "success";*/
}
// 发送文件切片到目标地址
public String sendFileSlice(String targetIp, FileSlice fileSlice) {
ResponseEntity<String> response = restTemplate.postForEntity("http://" + targetIp + "/data/put/", fileSlice, String.class);
if (!response.getStatusCode().is2xxSuccessful()) {
return "failed!";
}
return response.getBody();
}
public Set<FileSlice> getFileSlice(String targetIp, Long fileId) {
FileSlice[] fileSlices = restTemplate.getForObject("http://" + targetIp + "/data/get/" + fileId, FileSlice[].class);
HashSet<FileSlice> fileSliceSet = new HashSet<>();
fileSliceSet.addAll(Arrays.asList(fileSlices));
return fileSliceSet;
}
public String getFile(String fileName, Long fileId) {
/*ipList = getIpList(11113l,"dbc_server");
int sliceNum = ipList.length;*/
int sliceNum = ipList.length;
Set<FileSlice> result = new HashSet<>();
/* for (String ip : ipList) {
Set<FileSlice> fileSliceSet = getFileSlice(ip, fileId); //将fileId=1的碎片收回,HashSet无序收回
result.addAll(fileSliceSet);
}*/
//异常处理
for (int index = 0; index < ipList.length; index++) {
try {
Set<FileSlice> fileSliceSet = getFileSlice(ipList[index], fileId);
result.addAll(fileSliceSet);
}catch (Exception e){
}
}
List<FileSlice> list = new ArrayList<>(result);
SplitResult splitResult = new SplitResult();
splitResult.setFileSliceList(sortList(list));
ComputingResult computingResult = distributedComService.sliceComputing(splitResult);
MergeResult mergeResult = distributedComService.sliceMerge(computingResult,sliceNum);
try {
saveFile(fileName, mergeResult);
return "success";
} catch (Exception e) {
e.printStackTrace();
return "fail";
}
}
//给List集合里的元素进行排序
public List<FileSlice> sortList(List<FileSlice> list){
//List<FileSlice> list = new ArrayList<FileSlice>();
for (int i = 0; i < list.size(); i++) {
for (int j = list .size()-1; j > i; j--) {
int no= list.get(j).getSliceId();
int no_1= list.get(j-1).getSliceId();
if (no<no_1) {
//互换位置
FileSlice fil = list.get(j);
list.set(j, list.get(j-1));
list.set(j-1, fil );
}
}
}
return list;
}
public void saveFile(String fileName, MergeResult mergeResult) {
BufferedImage images = mergeResult.getImages();
//输出拼接后的图像
try {
ImageIO.write(images, "jpg", new File(directoryOutput + fileName));
} catch (IOException e) {
e.printStackTrace();
}
}
}
#项目端口
server.port=8999
#接入云端的端口设置
bishe.app.app-name=joliu
bishe.app.cloud-url=192.168.31.156:9090
#容器里的挂载目录
dbc.directory.output=/tmp/static/
#dbc.directory.output=/F:/resources/static/
#thymeleaf
spring.thymeleaf.prefix=classpath:/templates/
spring.thymeleaf.suffix=.html
spring.thymeleaf.mode=LEGACYHTML5
spring.thymeleaf.encoding=UTF-8
spring.thymeleaf.servlet.content-type=text/html
spring.thymeleaf.cache=false
# 设置文件上传的大小
spring.servlet.multipart.max-file-size=1024000000000MB
spring.servlet.multipart.max-request-size=1024000000000MB
\ No newline at end of file
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件显示</title>
</head>
<body>
<hr/>
<!--<a href="jpg/1.jpg">预览图片</a>-->
<!--<a href="@{/getFile/(fileName=${fileName},fileId=${fileId})}">预览图片</a>-->
<img th:src="file+'/'+${fileName}">
</body>
</html>
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件上传</title>
</head>
<body>
<hr/>
<form method="POST" enctype="multipart/form-data" action="/upload" id="uploadForm">
<p>
文件:<input type="file" name="file" />
</p>
<p>
文件ID:<input type="text" name="fileId" palcegolder="请输入" />
</p>
<p>
<input type="submit" value="上传" />
</p>
</form>
</body>
</html>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-business-cloud-cli</artifactId>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-commom-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-business-utils</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<!-- <dependency>
<groupId>top.ninwoo</groupId>
<artifactId>bishe-client-starter</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>-->
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
<version>2.3.0.RELEASE</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package top.ninwoo.dbc.cloud.client;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
//这个注解告诉我们,这是一个springboot项目
@SpringBootApplication
public class CloudClientStarter {
public static void main(String[] args) {
SpringApplication.run(CloudClientStarter.class,args);
}
}
package top.ninwoo.dbc.cloud.client.configure;
import top.ninwoo.dbc.api.service.DistributedComService;
import top.ninwoo.dbc.utils.FileServiceImplement;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
@Configuration
public class CloudClientCong {
@Primary
@Bean(name="cloudRestTemplates")
public RestTemplate restTemplate(ClientHttpRequestFactory factory){
return new RestTemplate(factory);
}
@Bean
public ClientHttpRequestFactory simpleClientHttpRequestFactory(){
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setReadTimeout(5000);//单位为ms
factory.setConnectTimeout(5000);//单位为ms
return factory;
}
@Bean
public DistributedComService distributedComService() {
return new FileServiceImplement();
}
}
package top.ninwoo.dbc.cloud.client.controller;
/*import top.ninwoo.dbc.api.po.FileSlice;
import top.ninwoo.dbc.client.service.FileService;
import org.springframework.web.bind.annotation.GetMapping;
import javax.annotation.Resource;
public class CloudClientController {
@Resource
private FileService fileService;
@GetMapping("test1")
public String sendData(String name) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileBytes(name.getBytes());
fileSlice.setFileId(1111L);
String result = fileService.sendFileSlice("127.0.0.1:8080", fileSlice);
return result;
}
@GetMapping("sendFile")
public String sendFile(String name, Long fileId) {
BufferedImage image = fileService.readImage(name);
return fileService.sendFile(fileId, image);
}
@GetMapping("getFile")
public String getFile(String name, Long fileId) {
return fileService.getFile(name, fileId);
}
}*/
package top.ninwoo.dbc.cloud.client.controller;
import top.ninwoo.dbc.cloud.client.service.FileComService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;
import top.ninwoo.dbc.cloud.client.service.FileComService;
import java.io.FileNotFoundException;
import java.io.IOException;
public class FilesController {
@Autowired
private FileComService fileComService;
@Value("${dbc.directory.output}")
private String directoryOutput;
/*文件上传*/
// 访问路径为:http://ip:port/upload
@RequestMapping(value = "/upload",method = RequestMethod.GET)
public String upload() {
return "fileUpload";
}
@RequestMapping(value = "/upload", method = RequestMethod.POST)
@ResponseBody
public String upload(@RequestParam("file") MultipartFile file, @RequestParam("fileId") Long fileId) {
if (!file.isEmpty()) {
try {
//BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(new File(file.getOriginalFilename())));
//FileOutputStream out = new FileOutputStream(new File(file.getOriginalFilename()));
byte[] bytes = file.getBytes();
fileComService.sendFile(fileId,bytes);
/* out.flush();
out.close();*/
} catch (FileNotFoundException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
} catch (IOException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
}
return "上传成功!";
} else {
return "上传失败,因为文件是空的.";
}
}
/*文件下载*/
//访问路径为:http://ip:port/download?name=x.jpg&fileId=1111
@RequestMapping("download")
public String view(String name, Long fileId, Model model) {
fileComService.getFile(name, fileId);
model.addAttribute("fileName", name);
return "fileDownload";
}
}
package top.ninwoo.dbc.cloud.client.service;
import top.ninwoo.dbc.api.po.*;
import top.ninwoo.dbc.api.service.DistributedComService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
@Service
public class FileComService {
@Value("${dbc.directory.output}")
private String directoryOutput;
@Qualifier(value="cloudRestTemplates")
@Autowired
private RestTemplate restTemplate;
@Autowired
private DistributedComService distributedComService;
String cloudIp = "127.0.0.1:8888";
/**
* 发送文件到云
* @param fileId
* @param bytes
* @return
*/
@Async("taskExecutor")
public String sendFile(Long fileId, byte[] bytes) {
// todo 这里需要制定文件的fileId
String result = sendFile(cloudIp, bytes);
if(result.equals("failed")) {
return "fail";
}
return "success";
}
// 发送文件切片到目标地址
public String sendFile(String targetIp, byte[] bytes) {
ResponseEntity<String> response = restTemplate.postForEntity("http://" + targetIp + "/data/put/", bytes, String.class);
if (!response.getStatusCode().is2xxSuccessful()) {
return "failed!";
}
return response.getBody();
}
public byte[] getFileBytes(String targetIp, Long fileId) {
byte[] bytes = restTemplate.getForObject("http://" + targetIp + "/data/get/" + fileId, byte[].class);
return bytes;
}
public String getFile(String fileName, Long fileId) {
byte[] fileBytes = getFileBytes(cloudIp,fileId);
CloudComputingResult cloudComputingResult = distributedComService.fileCloudComputing(fileBytes);
try {
saveFile(fileName, cloudComputingResult);
return "success";
} catch (Exception e) {
e.printStackTrace();
return "fail";
}
}
public void saveFile(String fileName, CloudComputingResult cloudComputingResult) {
byte[] bytes = cloudComputingResult.getFileComBytes();
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
try {
BufferedImage images = ImageIO.read(bis);
ImageIO.write(images, "jpg", new File(directoryOutput + fileName));
} catch (IOException e) {
e.printStackTrace();
}
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
/* BufferedImage images = mergeResult.getImages();
//输出拼接后的图像
try {
ImageIO.write(images, "jpg", new File(directoryOutput + fileName));
} catch (IOException e) {
e.printStackTrace();
}
*/
}
}
#项目端口
server.port=8000
#接入云端的端口设置
#bishe.app.app-name=joliu
#bishe.app.cloud-url=192.168.31.156:9090
#容器里的挂载目录
#dbc.directory.output=/tmp/static/
dbc.directory.output=/F:/resources/static/
#thymeleaf
spring.thymeleaf.prefix=classpath:/templates/
spring.thymeleaf.suffix=.html
spring.thymeleaf.mode=LEGACYHTML5
spring.thymeleaf.encoding=UTF-8
spring.thymeleaf.servlet.content-type=text/html
spring.thymeleaf.cache=false
# 设置文件上传的大小
spring.servlet.multipart.max-file-size=1024000000000MB
spring.servlet.multipart.max-request-size=1024000000000MB
\ No newline at end of file
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件显示</title>
</head>
<body>
<hr/>
<!--<a href="jpg/1.jpg">预览图片</a>-->
<!--<a href="@{/getFile/(fileName=${fileName},fileId=${fileId})}">预览图片</a>-->
<!--<img th:src="file+'/'+${fileName}">-->
</body>
</html>
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件上传(地面云计算)</title>
</head>
<body>
<hr/>
<form method="POST" enctype="multipart/form-data" action="/upload" id="uploadForm">
<p>
文件:<input type="file" name="file" />
</p>
<p>
文件ID:<input type="text" name="fileId" palcegolder="请输入" />
</p>
<p>
<input type="submit" value="上传" />
</p>
</form>
</body>
</html>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-business-cloud-server</artifactId>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-business-server</artifactId>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-commom-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-business-utils</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<!--这里写上main方法所在类的路径-->
<configuration>
<mainClass>top.ninwoo.dbc.server.ServerStarter</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package top.ninwoo.dbc.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ServerStarter {
public static void main(String[] args) {
SpringApplication.run(ServerStarter.class, args);
}
}
package top.ninwoo.dbc.server.controller;
import top.ninwoo.dbc.api.po.FileSlice;
import top.ninwoo.dbc.api.po.FileSliceComputing;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@RestController
public class ServerController {
/* @RequestMapping("/hello")
@ResponseBody
public String hello(){
return "hello,springboot";
}*/
private static final Map<Long, Map<Integer, FileSlice>> fileSliceMap = new HashMap<>();
@GetMapping("/hello/{name}/")
public String hello(@PathVariable(name = "name") String name) {
return "Hello " + name;
}
@PostMapping("/data/put/")
public String putData(@RequestBody FileSlice fileSlice) {
if(fileSlice == null) {
return "failed";
}
if(!fileSliceMap.containsKey(fileSlice.getFileId())) {
fileSliceMap.put(fileSlice.getFileId(), new HashMap<>());
}
Map<Integer, FileSlice> sliceMap = fileSliceMap.get(fileSlice.getFileId());
sliceMap.put(fileSlice.getSliceId(), fileSlice);
return "success";
}
@GetMapping("/data/get/{fileId}/{sliceId}")
public FileSlice getData(@PathVariable("fileId") Long fileId, @PathVariable("sliceId") int sliceId) {
try {
return fileSliceMap.get(fileId).get(sliceId);
} catch (Exception e) {
throw new RuntimeException("未找到对应的文件");
}
}
@GetMapping("/data/get/{fileId}")
public Set<FileSlice> getData(@PathVariable("fileId") Long fileId) {
try {
return new HashSet<FileSlice>(fileSliceMap.get(fileId).values());
} catch (Exception e) {
throw new RuntimeException("未找到文件");
}
}
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-business-utils</artifactId>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-commom-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package top.ninwoo.dbc.utils;
import top.ninwoo.dbc.api.po.*;
import top.ninwoo.dbc.api.service.DistributedComService;
import org.springframework.stereotype.Service;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Service
public class FileServiceImplement implements DistributedComService {
ImageToArrayUtils itau = new ImageToArrayUtils();
// 图片切分
@Override
public SplitResult fileSplit(BufferedImage bufferedImage, int sliceNum) {
/* // 读入大图
BufferedImage image = itau.readImage(srcOrig);*/
// 分割图片,并将其转化为byte数组
List<byte[]> byteList = itau.imageSplit(bufferedImage,sliceNum);
List<FileSlice> fileSliceList = new ArrayList<>();
for (int i = 0; i < byteList.size(); i++) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileId(1L);//文件的唯一标识
fileSlice.setSliceId(i);//数据碎片的id
fileSlice.setFileBytes(byteList.get(i));//文件的比特数据
fileSliceList.add(fileSlice);
}
SplitResult splitResult = new SplitResult();
splitResult.setFileSliceList(fileSliceList);
return splitResult;
}
//小图片处理---灰度处理
@Override
public ComputingResult sliceComputing(SplitResult splitResult){
List<FileSlice> fileSliceList = splitResult.getFileSliceList();
List<byte[]> grayByteList = itau.grayImage(fileSliceList);
//把每个小图片转为一个byte数组
//List<byte[]> grayByteList = itau.imageToByte(grayImage);
List<FileSliceComputing> grayfileSliceList = new ArrayList<>();
for (int i = 0; i < grayByteList.size(); i++) {
FileSliceComputing fileSliceComputing = new FileSliceComputing();
fileSliceComputing.setFileId(1L);//文件的唯一标识
fileSliceComputing.setSliceId(i);//数据碎片的id
fileSliceComputing.setFileBytes(grayByteList.get(i));//文件的比特数据
grayfileSliceList.add(fileSliceComputing);
}
ComputingResult computingResult = new ComputingResult();
computingResult.setFileSliceComputingList(grayfileSliceList);
return computingResult;
}
@Override
public MergeResult sliceMerge(ComputingResult computingResult,int sliceNum) {
List<FileSliceComputing> grayfileSliceList = computingResult.getFileSliceComputingList();
BufferedImage finalImg = itau.imageMerge(grayfileSliceList, sliceNum);
MergeResult mergeResult = new MergeResult();
mergeResult.setImages(finalImg);
return mergeResult;
}
public SendResult sendFileSlice(String targetIp, FileSlice fileSlice) {
return null;
}
public FileSlice getFileSlice(String targetIp, Long fileId, int sliceId) {
return null;
}
//地面云计算处理图片
@Override
public CloudComputingResult fileCloudComputing(byte[] bytes){
//1、byte[]转为BufferedImage
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
BufferedImage image = null;
try {
image = ImageIO.read(bis);
} catch (IOException e) {
e.printStackTrace();
}
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
//2、BufferedImage进行灰度化处理
int width = image.getWidth();
int height = image.getHeight();
BufferedImage grayImage = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);//重点,技巧在这个参数BufferedImage.TYPE_BYTE_GRAY
for(int i= 0 ; i < width ; i++){
for(int j = 0 ; j < height; j++){
int rgb = image.getRGB(i, j);
grayImage.setRGB(i, j, rgb);
}
}
//3、灰度化处理后的BufferedImage转为byte[]
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try {
ImageIO.write(grayImage,"jpg",bos);
} catch (IOException e) {
e.printStackTrace();
}
try {
bos.close();
} catch (IOException e) {
e.printStackTrace();
}
byte[] fileComBytes = bos.toByteArray();
CloudComputingResult cloudComputingResult = new CloudComputingResult();
cloudComputingResult.setFileComBytes(fileComBytes);
return cloudComputingResult;
}
}
package top.ninwoo.dbc.utils;
import top.ninwoo.dbc.api.po.FileSlice;
import top.ninwoo.dbc.api.po.FileSliceComputing;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class ImageToArrayUtils {
public static int rows = 1 ;
//public static int sliceNum ;
//将image读为BufferedImage
/* public static BufferedImage readImage(String fileName){
//File file = new File(directoryInput+fileName);
File file = new File(fileName);
FileInputStream fis = null;
BufferedImage image = null;
try {
fis = new FileInputStream(file);
image = ImageIO.read(fis);
} catch (Exception e) {
if (fis != null) {
try {
fis.close();
} catch (IOException ioException) {
ioException.printStackTrace();
}
}
throw new RuntimeException("文件读取错误" + e.getMessage());
}
return image;
}*/
public static List<byte[]> imageSplit(BufferedImage images, int sliceNum){
//BufferedImage images = readImage(name);
int cols = sliceNum / rows;
// 计算每个小图的宽度和高度
int chunkWidth = images.getWidth() / cols;
int chunkHeight = images.getHeight() / rows;
int count = 0;
//BufferedImage imgs[] = new BufferedImage[chunks];
BufferedImage imgs[] = new BufferedImage[sliceNum];
for (int x = 0; x < rows; x++) {
for (int y = 0; y < cols; y++) {
//设置小图的大小和类型
imgs[count] = new BufferedImage(chunkWidth, chunkHeight, images.getType());
//写入图像内容
Graphics2D gr = imgs[count++].createGraphics();
gr.drawImage(images, 0, 0,
chunkWidth, chunkHeight,
chunkWidth* y, chunkHeight * x,
chunkWidth * y + chunkWidth,
chunkHeight * x + chunkHeight, null);
gr.dispose();
}
}
//将imgs[]转为相应的byte数组
List<byte[]> byteList = new ArrayList<byte[]>();
for (int i = 0; i < imgs.length; i++) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try {
ImageIO.write(imgs[i],"jpg",bos);
} catch (IOException e) {
e.printStackTrace();
}
try {
bos.close();
} catch (IOException e) {
e.printStackTrace();
}
byteList.add(bos.toByteArray());
}
// 输出小图
/* for (int i = 0; i < imgs.length; i++) {
try {
ImageIO.write(imgs[i], "jpg", new File("F:\\images\\split\\image" + i + ".jpg"));
} catch (IOException e) {
e.printStackTrace();
}
}*/
return byteList;
}
//图片灰度化,同时将grayImage[]转为byte[]数组
public static List<byte[]> grayImage(List<FileSlice> fileSliceList){
BufferedImage images[] = new BufferedImage[fileSliceList.size()];
BufferedImage grayImage[] = new BufferedImage[fileSliceList.size()];
int[] width = new int[fileSliceList.size()];
int[] height = new int[fileSliceList.size()];
for (int i = 0; i < fileSliceList.size(); i++) {
ByteArrayInputStream bis = new ByteArrayInputStream(fileSliceList.get(i).getFileBytes());
try {
images[i] = ImageIO.read(bis);
width[i] = images[i].getWidth();
height[i] = images[i].getHeight();
grayImage[i] = new BufferedImage(width[i], height[i], BufferedImage.TYPE_BYTE_GRAY);//重点,技巧在这个参数BufferedImage.TYPE_BYTE_GRAY
for(int j= 0 ; j < width[i] ; j++){
for(int k = 0 ; k < height[i]; k++){
int rgb = images[i].getRGB(j, k);
grayImage[i].setRGB(j, k, rgb);
}
}
ImageIO.write(grayImage[i], "jpg", new File("\\tmp\\static\\image" + i + ".jpg"));
} catch (IOException e) {
e.printStackTrace();
}
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
List<byte[]> byteList = new ArrayList<>();
for (int i = 0; i < grayImage.length; i++) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try {
ImageIO.write(grayImage[i],"jpg",bos);
} catch (IOException e) {
e.printStackTrace();
}
try {
bos.close();
} catch (IOException e) {
e.printStackTrace();
}
byteList.add(bos.toByteArray());
}
return byteList;
}
//小图片灰度化后,进行合并
public static BufferedImage imageMerge(List<FileSliceComputing> fileSliceComputingList,int sliceNum){
//byte[]--->BufferedImage
BufferedImage images[] = new BufferedImage[fileSliceComputingList.size()];
for (int i = 0; i < fileSliceComputingList.size(); i++) {
ByteArrayInputStream bis = new ByteArrayInputStream(fileSliceComputingList.get(i).getFileBytes());
try {
images[i] = ImageIO.read(bis);
//ImageIO.write(images[i], "jpg", new File("F:\\images\\grayImage\\image" + i + ".jpg"));
ImageIO.write(images[i], "jpg", new File("\\tmp\\static\\image" + i + ".jpg"));
} catch (IOException e) {
e.printStackTrace();
}
try {
bis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
int cols = sliceNum/rows;
int chunks = sliceNum;
int chunkWidth, chunkHeight;
int type;
//读入小图
File[] imgFiles = new File[chunks];
for (int i = 0; i < chunks; i++) {
//imgFiles[i] = new File("F:\\images\\grayImage\\image" + i + ".jpg");
imgFiles[i] = new File("\\tmp\\static\\image" + i + ".jpg");
}
//创建BufferedImage
BufferedImage[] buffImages = new BufferedImage[chunks];
for (int i = 0; i < chunks; i++) {
try {
buffImages[i] = ImageIO.read(imgFiles[i]);
} catch (IOException e) {
e.printStackTrace();
}
}
type = buffImages[0].getType();
chunkWidth = buffImages[0].getWidth();
chunkHeight = buffImages[0].getHeight();
//设置拼接后图的大小和类型
BufferedImage finalImg = new BufferedImage(chunkWidth * cols, chunkHeight * rows, type);
//写入图像内容
int num = 0;
for (int i = 0; i < rows; i++) {
for (int j = 0; j < cols; j++) {
finalImg.createGraphics().drawImage(buffImages[num], chunkWidth * j, chunkHeight * i, null);
num++;
}
}
//输出拼接后的图像
/* try {
ImageIO.write(finalImg, "jpeg", new File("D:\\vx-ymwang\\images\\mergeImage\\finalImg.jpg"));
} catch (IOException e) {
e.printStackTrace();
}*/
return finalImg;
}
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-business-computing</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dbc-commom-api</artifactId>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package top.ninwoo.dbc.api.po;
import lombok.Data;
@Data
public class CloudComputingResult {
private byte[] fileComBytes;
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
import java.util.List;
@Data
public class ComputingResult {
private List<FileSliceComputing> fileSliceComputingList;
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
/**
* @author ymwang
* 文件的切片类
*/
@Data
public class FileSlice {
// 文件的唯一id
private Long fileId;
// 切片的id,唯一且有顺序要求
private int sliceId;
// 文件的比特数据
private byte[] fileBytes;
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
/**
* @author ymwang
* 文件的切片类
*/
@Data
public class FileSliceComputing {
// 文件的唯一id
private Long fileId;
// 切片的id,唯一且有顺序要求
private int sliceId;
// 文件的比特数据
private byte[] fileBytes;
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
import java.awt.image.BufferedImage;
@Data
public class MergeResult {
//private byte[] fileBytes;
private BufferedImage images;
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
@Data
public class SendResult {
}
package top.ninwoo.dbc.api.po;
import lombok.Data;
import java.util.List;
@Data
public class SplitResult {
private List<FileSlice> fileSliceList;
}
package top.ninwoo.dbc.api.service;
import top.ninwoo.dbc.api.po.*;
import java.awt.image.BufferedImage;
public interface DistributedComService {
/**
* 文件切片
* @param bufferedImage 图片
* @param sliceNum 切片数量
* @return
*/
SplitResult fileSplit(BufferedImage bufferedImage, int sliceNum);
//SplitResult fileSplit(String fileName, int sliceNum);
/**
* 切片处理
* @param splitResult
* @return
*/
ComputingResult sliceComputing(SplitResult splitResult);
/**
* 切片聚合接口
* @param computingResult
* @param sliceNum 切片数量
* @return
*/
MergeResult sliceMerge(ComputingResult computingResult, int sliceNum);
/**
* 切片发送服务
* @param targetIp
* @param fileSlice
* @return
*/
SendResult sendFileSlice(String targetIp, FileSlice fileSlice);
/**
* 指定地址的ip获取切片
* @param targetIp
* @param fileId
* @param sliceId
* @return
*/
FileSlice getFileSlice(String targetIp, Long fileId, int sliceId);
/**
* 图片处理(地面云计算)
* @param bytes
* @return
*/
CloudComputingResult fileCloudComputing(byte[] bytes);
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<!--删掉root文件下的src文件夹后,添加的语句-->
<packaging>pom</packaging>
<modules>
<module>dbc-business-client</module>
<module>dbc-business-cloud-cli</module>
<module>dbc-business-cloud-server</module>
<module>dbc-business-server</module>
<module>dbc-business-utils</module>
<module>dbc-commom-api</module>
</modules>
<artifactId>cnf-distributed-business-computing</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-commom-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dbc-business-client</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
<version>${lombok.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
<scope>compile</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
<version>${thymeleaf.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${springboot.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<!-- <version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>-->
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**
!**/src/test/**
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
### VS Code ###
.vscode/
FROM centos
MAINTAINER Joliu<ljo0412@live.com>
RUN yum install -y http://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm &&\
yum install -y iperf iptables tc
RUN yum install -y net-tools.x86_64
RUN echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-security main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-updates main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-proposed main restricted universe multiverse
deb http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-security main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-updates main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-proposed main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu-old-releases/ubuntu/ xenial-backports main restricted universe multiverse
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-file-transfer</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dfs-common-api</artifactId>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.ujmp</groupId>
<artifactId>ujmp-core</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package top.ninwoo.commonapi.po;
import lombok.Data;
/**
* @author ljo
* 文件的切片类
*/
@Data
public class FileSlice {
// 文件的唯一id
private Long fileId;
// 切片的id,唯一且有顺序要求
private int sliceId;
// 索引的偏移量
private Long start;
// 文件比特数据的有效长度 - 这个值设置的原因是 fileBytes可能有补位的情况。
private int size;
/* private Long size;*/
// 文件的比特数据
private byte[] fileBytes;
}
package top.ninwoo.commonapi.po;
import lombok.Builder;
import lombok.Getter;
import java.math.BigDecimal;
@Builder
@Getter
public class Fraction {
private BigDecimal mole;
private BigDecimal deno;
private boolean negate = false;
}
package top.ninwoo.commonapi.po;
import lombok.Data;
@Data
public class MergeResult extends Result {
private byte[] fileBytes;
public byte[] getFileBytes() {
return fileBytes;
}
public void setFileBytes(byte[] fileBytes) {
this.fileBytes = fileBytes;
}
}
package top.ninwoo.commonapi.po;
import lombok.Data;
@Data
public class Result {
private boolean success;
private String errMsg;
}
package top.ninwoo.commonapi.po;
import lombok.Data;
@Data
public class SendResult extends Result {
}
package top.ninwoo.commonapi.po;
import lombok.Data;
import java.util.List;
@Data
public class SplitResult extends Result {
private List<FileSlice> fileSliceList;
}
package top.ninwoo.commonapi.service;
import top.ninwoo.commonapi.po.FileSlice;
import top.ninwoo.commonapi.po.MergeResult;
import top.ninwoo.commonapi.po.SendResult;
import top.ninwoo.commonapi.po.SplitResult;
/**
* @author ljo
* 用于发送文件的接口
*/
public interface TransferService {
/**
* 文件切片
* @param fileBytes
* @param sliceNum 切片数量(经过冗余之后的数量)
* @param origNum 原始数组切分的数量
* @return
*/
SplitResult fileSplit(byte[] fileBytes, int sliceNum, int origNum);
/**
* 切片聚合接口
* @param splitResult
* @param sliceNum 切片数量(经过冗余之后的数量)
* @param sliceNum origNum 原始数组切分的数量
* @return
*/
MergeResult sliceMerge(SplitResult splitResult, int sliceNum, int origNum);
/**
* 切片发送服务
* @param targetIp
* @param fileSlice
* @return
*/
SendResult sendFileSlice(String targetIp, FileSlice fileSlice);
/**
* 指定地址的ip获取切片
* @param targetIp
* @param fileId
* @param sliceId
* @return
*/
FileSlice getFileSlice(String targetIp, Long fileId, int sliceId);
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-file-transfer</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dfs-file-utils</artifactId>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-common-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.ujmp/ujmp-core -->
<dependency>
<groupId>org.ujmp</groupId>
<artifactId>ujmp-core</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package top.ninwoo.app.dfs.service;
import top.ninwoo.commonapi.po.*;
import top.ninwoo.commonapi.service.TransferService;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class FileTransferServiceImpl implements TransferService {
@Override
public SplitResult fileSplit(byte[] fileBytes, int sliceNum, int origNum) {
EncodeAndDecode encode = new EncodeAndDecode();
/*1、将传进来的byte数组拆分,也即原数组等大小拆分,同时转为BigInteger型,并将其转置成m*1型*/
BigInteger[][] filebytesBigInt = encode.fileByteSplit(fileBytes,origNum);
//encode.printMartixInt(filebytesBigInt);
/*2 将filebytes(BigInteger型)转为BigDecimal型*/
BigDecimal[][] filebytesBigDec = encode.bigIntToDecimal(filebytesBigInt);
/*3、创建编码矩阵---上半部分的单位矩阵+下半部分的范德蒙矩阵*/
BigDecimal[][] encodeDecimal = encode.encodeMatrix(origNum,sliceNum-origNum);
/*4、生成编码数据块encodeDataMatrix(BigInteger型数组(m+shards)*1)*/
//两个BigInteger型数组相乘
BigDecimal[][] encodeDataMatrixBigDec = encode.encodeData(encodeDecimal,filebytesBigDec);
/*5、将BigDecimal转为BigInteger*/
BigInteger[] encodeDataMatrixTran = encode.bigDecToInt(encodeDataMatrixBigDec);
/*6、将数据碎片数组包装成方法返回值类型*/
List<FileSlice> fileSliceList = new ArrayList<>();
for (int j = 0; j < encodeDataMatrixTran.length; j++) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileId(1L);//文件的唯一标识
fileSlice.setSliceId(j);//数据碎片的id
fileSlice.setSize(fileBytes.length);//文件比特数据的有效长度
fileSlice.setFileBytes(encodeDataMatrixTran[j].toByteArray());//文件的比特数据
fileSliceList.add(fileSlice);
}
SplitResult splitResult = new SplitResult();
splitResult.setFileSliceList(fileSliceList);
return splitResult;//这是最后要返回的东西,也就是图片分割的结果
}
@Override
public MergeResult sliceMerge(SplitResult splitResult, int sliceNum, int origNum) {
EncodeAndDecode encode = new EncodeAndDecode();
List<FileSlice> fileSliceListInput = splitResult.getFileSliceList();
BigInteger[][] encodeDataBigInt = new BigInteger[fileSliceListInput.size()][1]; //从fileSliceList中任取m个数据碎片---7
for (int i = 0; i < fileSliceListInput.size(); i++) {
encodeDataBigInt[i][0] = new BigInteger(fileSliceListInput.get(i).getFileBytes());
}
//encodeDataBigInt转为encodeDataBigDec
BigDecimal[][] encodeDataBigDec = encode.bigIntToDecimal(encodeDataBigInt);
/*从编码矩阵中取对应的行,形成新的encodeMatrixNew*/
BigDecimal[][] encodeMatrix = encode.encodeMatrix(origNum,sliceNum-origNum);
BigDecimal[][] encodeMatrixNew = new BigDecimal[fileSliceListInput.size()][fileSliceListInput.size()];
for (int i = 0; i < encodeMatrixNew.length; i++) {
for (int j = 0; j < encodeMatrixNew[0].length; j++) {
encodeMatrixNew[i][j] = encodeMatrix[fileSliceListInput.get(i).getSliceId()][j];
}
}
/*求新的编码矩阵的逆矩阵*/
Fraction[][] encodeMatrixNewInv = encode.getReverseMartrix(encodeMatrixNew);
/*两矩阵相乘*/
BigDecimal[][] origDataBigDec = encode.encodeData(encodeMatrixNewInv,encodeDataBigDec);
/*BigDecimal转BigInteger*/
BigInteger[] origDataBigInt = encode.bigDecToInt(origDataBigDec);
/*origDataBigInt型数组转为byte数组(origDataBytes)*/
byte[] fileByteDest = encode.bigIntegerToBytes(origDataBigInt,fileSliceListInput.get(0).getSize());
MergeResult mergeResult = new MergeResult();
mergeResult.setFileBytes(fileByteDest);
return mergeResult;
}
@Override
public SendResult sendFileSlice(String targetIp, FileSlice fileSlice) {
return null;
}
@Override
public FileSlice getFileSlice(String targetIp, Long fileId, int sliceId) {
return null;
}
}
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-file-transfer</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dfs-transfer-client</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-common-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-file-utils</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>cnf-client-starter</artifactId>
<!--<version>1.0-SNAPSHOT</version>-->
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<!--这里写上main方法所在类的路径-->
<configuration>
<mainClass>top.ninwoo.app.dfs.client.ClientStart</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package top.ninwoo.app.dfs.client;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
@SpringBootApplication
@EnableAsync
public class ClientStart {
public static void main(String[] args) {
SpringApplication.run(ClientStart.class, args);
}
}
\ No newline at end of file
package top.ninwoo.app.dfs.client.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
@Configuration
@EnableAsync // 开启异步调用 多线程
public class AsyncTaskConfig{
@Bean("taskExecutor")
public Executor taskExecutor() {
// 新建一个任务执行器
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setCorePoolSize(10);////核心线程池大小
taskExecutor.setMaxPoolSize(30);// 设置最大的线程数量
taskExecutor.setQueueCapacity(25);// 等待队列
taskExecutor.initialize();// 如果不初始化,导致找不到执行器
return taskExecutor;
}
}
package top.ninwoo.app.dfs.client.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import top.ninwoo.app.dfs.service.FileTransferServiceImpl;
import top.ninwoo.commonapi.service.TransferService;
@Configuration
public class MainConfig {
@Primary
@Bean(name="fileRestTemplate")
public RestTemplate restTemplate(ClientHttpRequestFactory factory){
return new RestTemplate(factory);
}
@Bean
public ClientHttpRequestFactory simpleClientHttpRequestFactory(){
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setReadTimeout(20000);//单位为ms
factory.setConnectTimeout(20000);//单位为ms
return factory;
}
@Bean
public TransferService transferService() {
return new FileTransferServiceImpl();
}
}
package top.ninwoo.app.dfs.client.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
@Configuration
public class StaticResourceConfiguration extends WebMvcConfigurerAdapter {
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
registry.addResourceHandler("/file/**").addResourceLocations("file:/tmp/static/");
//registry.addResourceHandler("/file/**").addResourceLocations("file:/D:/vx-ymwang/resoueces/static/");
super.addResourceHandlers(registry);
}
}
package top.ninwoo.app.dfs.client.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;
import top.ninwoo.app.dfs.client.service.FileService;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@Controller
public class FileController {
@Autowired
private FileService fileService;
/*文件上传*/
// 访问路径为:http://ip:port/upload
@RequestMapping(value = "/upload", method = RequestMethod.GET)
public String upload() {
return "fileUpload";
}
/*
public ModelAndView upload() {
return new ModelAndView("fileUpload");
}
*/
@RequestMapping(value = "/upload", method = RequestMethod.POST)
@ResponseBody
public String upload(@RequestParam("file") MultipartFile file, @RequestParam("fileId") Long fileId) {
if (!file.isEmpty()) {
try {
//BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(new File(file.getOriginalFilename())));
FileOutputStream out = new FileOutputStream(new File(file.getOriginalFilename()));
byte[] bytes = file.getBytes();
fileService.sendFile(fileId,bytes);
//out.write(file.getBytes());
out.flush();
out.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
} catch (IOException e) {
e.printStackTrace();
return "上传失败," + e.getMessage();
}
return "上传成功!";
} else {
return "上传失败,因为文件是空的.";
}
}
/*文件下载*/
// 访问路径为:http://ip:port/upload
/* @RequestMapping(value = "/view", method = RequestMethod.GET)
public String download() {
return "fileFind";
}*/
/*文件下载*/
//访问路径为:http://ip:port/download?name=x.jpg&fileId=1111
@RequestMapping("download")
public String view(String name, Long fileId, Model model) {
fileService.getFile(name, fileId);
model.addAttribute("fileName", name);
return "fileDownload";
}
}
package top.ninwoo.app.dfs.client.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import top.ninwoo.app.dfs.client.service.FileService;
import top.ninwoo.commonapi.po.FileSlice;
@RestController
public class TestController {
@Autowired
private FileService fileService;
@RequestMapping("/hello")
@ResponseBody
public String hello(){
return "hello,SpringBoot";
}
@GetMapping("test1")
public String sendData(String name) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileBytes(name.getBytes());
fileSlice.setFileId(1111L);
fileSlice.setSize(name.getBytes().length);
fileSlice.setStart(0L);
/*String result = fileService.sendFileSlice("127.0.0.1:8080", fileSlice);*/
String result = fileService.sendFileSlice("172.17.3", fileSlice);
return result;
}
/* @GetMapping("sendFile")
public String sendFile(String name, Long fileId) {
byte[] bytes = fileService.readFile(name);
return fileService.sendFile(fileId, bytes);
}*/
@GetMapping("getFile")
public String getFile(String name, Long fileId) {
return fileService.getFile(name, fileId);
}
}
package top.ninwoo.app.dfs.client.service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import top.ninwoo.bishe.starter.service.NetworkService;
import top.ninwoo.commonapi.po.FileSlice;
import top.ninwoo.commonapi.po.MergeResult;
import top.ninwoo.commonapi.po.SplitResult;
import top.ninwoo.commonapi.service.TransferService;
import javax.annotation.PostConstruct;
import java.io.FileOutputStream;
import java.util.*;
/**
* 文件服务
*/
@Service
public class FileService {
@Value("${dfs.directory.output}")
private String directoryOutput;
@Qualifier(value="fileRestTemplate")
@Autowired
private RestTemplate restTemplate;
@Autowired
private NetworkService networkService;
@Autowired
private TransferService transferService;
private String[] ipList=null;
@PostConstruct
public void init() {
ipList = getIpList(11112l,"dfs_server");
System.out.println(Arrays.toString(ipList));
}
/*获取容器ip*/
/* String[] ipList = new String[]{"127.0.0.1:8080","127.0.0.1:8081","127.0.0.1:8082","127.0.0.1:8083","127.0.0.1:8084",
"127.0.0.1:8085","127.0.0.1:8086","127.0.0.1:8087", "127.0.0.1:8088","127.0.0.1:8089",
"127.0.0.1:8180","127.0.0.1:8181","127.0.0.1:8182","127.0.0.1:8183","127.0.0.1:8184"};*/
public String[] getIpList(Long clusterId, String appName) {
List<String> ipListS = networkService.getIpListByAppName(clusterId, appName);//用于存储容器ip的list集合
String[] containerIp = new String[ipListS.size()];//将集合中的元素存在数组里,因为下边用的是数组
for (int i = 0; i < ipListS.size(); i++) {
if(!ipListS.isEmpty()){
String ip_tmp = ipListS.get(i);
String[] split_list = ip_tmp.split("/");
containerIp[i] = split_list[0]+":8084";
}
}
return containerIp;
}
/**
* 发送文件到分布式节点
* @param fileId
* @param fileBytes
* @return
*/
@Async("taskExecutor")
public String sendFile(Long fileId, byte[] fileBytes) {
// 通过集群服务器接口获取当前集群的节点数量
ipList = this.getIpList(11112l,"dfs_server");
int sliceNum = ipList.length;
//int sliceNum = 30;
int origNum = 6;
// todo 这里需要制定文件的fileId
SplitResult splitResult = transferService.fileSplit(fileBytes, sliceNum, origNum);
//顺序,依次发送碎片到各个服务器
int ipIndex = 0;
for (FileSlice slice : splitResult.getFileSliceList()) {
slice.setFileId(fileId);
while(true){
try {
sendFileSlice(ipList[ipIndex], slice);
ipIndex = (ipIndex + 1) % ipList.length;
break;
}catch (Exception e){
ipIndex = (ipIndex + 1) % ipList.length;
}
}
}
return "success";
}
// 发送文件切片到目标地址
public String sendFileSlice(String targetIp, FileSlice fileSlice) {
ResponseEntity<String> response = restTemplate.postForEntity("http://" + targetIp + "/data/put/", fileSlice, String.class);
if (!response.getStatusCode().is2xxSuccessful()) {
return "failed!";
}
return response.getBody();
}
public Set<FileSlice> getFileSlice(String targetIp, Long fileId) {
FileSlice[] fileSlices = restTemplate.getForObject("http://" + targetIp + "/data/get/" + fileId, FileSlice[].class);
HashSet<FileSlice> fileSliceSet = new HashSet<>();
fileSliceSet.addAll(Arrays.asList(fileSlices));
return fileSliceSet;
}
public String getFile(String fileName, Long fileId) {
ipList = this.getIpList(11112l,"dfs_server");
int sliceNum = ipList.length;
//int sliceNum = 30;
int origNum = 6;
Set<FileSlice> result = new HashSet<>();
//异常处理
for (int index = 0; index < ipList.length; index++) {
try {
Set<FileSlice> fileSliceSet = getFileSlice(ipList[index], fileId);
result.addAll(fileSliceSet);
}catch (Exception e){
}
if(result.size() >= origNum) {
break;
}
}
//System.out.println("result:"+result.size());
/*for (String ip : ipList) {
Set<FileSlice> fileSliceSet = getFileSlice(ip, fileId);
result.addAll(fileSliceSet);
if(result.size() >= origNum) {
break;
}
}*/
if(result.size() < origNum) {
throw new RuntimeException("不可恢复的文件");
}
SplitResult splitResult = new SplitResult();
splitResult.setFileSliceList(new ArrayList<>(result));
MergeResult mergeResult = transferService.sliceMerge(splitResult, sliceNum, origNum);
try {
saveFile(fileName, mergeResult);
return "success";
} catch (Exception e) {
e.printStackTrace();
return "fail";
}
}
public void saveFile(String fileName, MergeResult mergeResult) throws Exception {
byte[] fileByte = mergeResult.getFileBytes();
//将字节数组转为文件----可以验证读取图片到byte数组是否正确
FileOutputStream fos = new FileOutputStream(directoryOutput + fileName);
/*FileOutputStream fos = new FileOutputStream("F:/恢复.PNG");*/
fos.write(fileByte, 0, fileByte.length);
fos.close();
}
}
\ No newline at end of file
server.port=8099
bishe.app.app-name=joliu
bishe.app.cloud-url=192.168.31.156:9090
dfs.directory.output=/tmp/static/
#dfs.directory.output=/D:/vx-ymwang/resoueces/static/
#thymeleaf
spring.thymeleaf.prefix=classpath:/templates/
spring.thymeleaf.suffix=.html
spring.thymeleaf.mode=LEGACYHTML5
spring.thymeleaf.encoding=UTF-8
spring.thymeleaf.servlet.content-type=text/html
spring.thymeleaf.cache=false
# ļϴĴС
spring.servlet.multipart.max-file-size=1024000000000MB
spring.servlet.multipart.max-request-size=1024000000000MB
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件显示</title>
</head>
<body>
<hr/>
<!--<a href="jpg/1.jpg">预览图片</a>-->
<!--<a href="@{/getFile/(fileName=${fileName},fileId=${fileId})}">预览图片</a>-->
<img th:src="file+'/'+${fileName}">
</body>
</html>
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>文件上传</title>
</head>
<body>
<hr/>
<form method="POST" enctype="multipart/form-data" action="/upload" id="uploadForm">
<p>
文件:<input type="file" name="file" />
</p>
<p>
文件ID:<input type="text" name="fileId" palcegolder="请输入" />
</p>
<p>
<input type="submit" value="上传" />
</p>
</form>
</body>
</html>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf-distributed-file-transfer</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dfs-transfer-server</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-common-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.ujmp/ujmp-core -->
<dependency>
<groupId>org.ujmp</groupId>
<artifactId>ujmp-core</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<!--这里写上main方法所在类的路径-->
<configuration>
<mainClass>top.ninwoo.app.dfs.ServerStarter</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package top.ninwoo.app.dfs;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ServerStarter {
public static void main(String[] args) {
SpringApplication.run(ServerStarter.class, args);
}
}
package top.ninwoo.app.dfs.controller;
import org.springframework.web.bind.annotation.*;
import top.ninwoo.commonapi.po.FileSlice;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@RestController
public class FileSliceTranferController {
private static final Map<Long, Map<Integer, FileSlice>> fileSliceMap = new HashMap<>();
@GetMapping("/hello/{name}/")
public String hello(@PathVariable(name = "name") String name) {
return "Hello " + name;
}
@PostMapping("/data/put/")
public String putData(@RequestBody FileSlice fileSlice) {
if(fileSlice == null) {
return "failed";
}
if(!fileSliceMap.containsKey(fileSlice.getFileId())) {
fileSliceMap.put(fileSlice.getFileId(), new HashMap<>());
}
Map<Integer, FileSlice> sliceMap = fileSliceMap.get(fileSlice.getFileId());
sliceMap.put(fileSlice.getSliceId(), fileSlice);
return "success";
}
@GetMapping("/data/get/{fileId}/{sliceId}")
public FileSlice getData(@PathVariable("fileId") Long fileId, @PathVariable("sliceId") int sliceId) {
try {
return fileSliceMap.get(fileId).get(sliceId);
} catch (Exception e) {
throw new RuntimeException("未找到对应的文件");
}
}
@GetMapping("/data/get/{fileId}")
public Set<FileSlice> getData(@PathVariable("fileId") Long fileId) {
try {
return new HashSet<FileSlice>(fileSliceMap.get(fileId).values());
} catch (Exception e) {
throw new RuntimeException("未找到文件");
}
}
}
\ No newline at end of file
package top.ninwoo.app.dfs.repository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Repository;
import top.ninwoo.commonapi.po.FileSlice;
import java.io.*;
import java.util.Arrays;
@Repository
@Slf4j
public class FileSliceRepository {
/*private static final String PATH = "E:\\code\\bs-app\\distributed-file-transfer\\output\\";*/
private static final String PATH = "F:\\output\\";
private static final int DEFAULT_LENGTH = 1000;
public FileSlice getFileSlice(Long fileId, int sliceId) {
FileSlice fileSlice = new FileSlice();
fileSlice.setSliceId(sliceId);
fileSlice.setFileId(fileId);
fileSlice.setStart(0L);
ByteArrayOutputStream out = null;
try {
InputStream in = new FileInputStream(PATH + fileId + "-" + sliceId + ".data");
out = new ByteArrayOutputStream();
byte[] buffer = new byte[1024 * 4];
int n = 0;
while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n);
}
} catch (Exception e) {
e.printStackTrace();
}
byte[] bytes = Arrays.copyOf(out.toByteArray(), DEFAULT_LENGTH);
fileSlice.setFileBytes(bytes);
fileSlice.setSize(out.toByteArray().length);
return fileSlice;
}
public void saveFileSlice(FileSlice slice) {
OutputStream os = null;
try {
File file = new File(PATH + slice.getFileId() + "-" + slice.getSliceId() + ".data");
file.createNewFile();
os = new FileOutputStream(file);
byte[] targetBytes = Arrays.copyOf(slice.getFileBytes(), slice.getSize());
os.write(targetBytes);
os.flush();
} catch (Exception e) {
log.error("save fileSlice Error:{}", e.getMessage());
} finally {
if (os != null) {
try {
os.close();
} catch (Exception e) {
log.error("file close error!");
}
}
}
}
}
package top.ninwoo.app.dfs.service;
import org.springframework.stereotype.Service;
import top.ninwoo.app.dfs.util.Fraction;
import top.ninwoo.commonapi.po.FileSlice;
import top.ninwoo.commonapi.po.MergeResult;
import top.ninwoo.commonapi.po.SendResult;
import top.ninwoo.commonapi.po.SplitResult;
import top.ninwoo.commonapi.service.TransferService;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
@Service
public class FileTransferServiceImpl implements TransferService {
@Override
public SplitResult fileSplit(byte[] fileBytes, int sliceNum, int origNum) {
EncodeAndDecode encode = new EncodeAndDecode();
/*1、将传进来的byte数组拆分,也即原数组等大小拆分,同时转为BigInteger型,并将其转置成m*1型*/
BigInteger[][] filebytesBigInt = encode.fileByteSplit(fileBytes,origNum);
//encode.printMartixInt(filebytesBigInt);
/*2 将filebytes(BigInteger型)转为BigDecimal型*/
BigDecimal[][] filebytesBigDec = encode.bigIntToDecimal(filebytesBigInt);
/*3、创建编码矩阵---上半部分的单位矩阵+下半部分的范德蒙矩阵*/
BigDecimal[][] encodeDecimal = encode.encodeMatrix(origNum,sliceNum-origNum);
/*4、生成编码数据块encodeDataMatrix(BigInteger型数组(m+shards)*1)*/
//两个BigInteger型数组相乘
BigDecimal[][] encodeDataMatrixBigDec = encode.encodeData(encodeDecimal,filebytesBigDec);
/*5、将BigDecimal转为BigInteger*/
BigInteger[] encodeDataMatrixTran = encode.bigDecToInt(encodeDataMatrixBigDec);
/*6、将数据碎片数组包装成方法返回值类型*/
List<FileSlice> fileSliceList = new ArrayList<>();
for (int j = 0; j < encodeDataMatrixTran.length; j++) {
FileSlice fileSlice = new FileSlice();
fileSlice.setFileId(1L);//文件的唯一标识
fileSlice.setSliceId(j);//数据碎片的id
fileSlice.setSize(fileBytes.length);//文件比特数据的有效长度
fileSlice.setFileBytes(encodeDataMatrixTran[j].toByteArray());//文件的比特数据
fileSliceList.add(fileSlice);
}
SplitResult splitResult = new SplitResult();
splitResult.setFileSliceList(fileSliceList);
return splitResult;//这是最后要返回的东西,也就是图片分割的结果
}
@Override
public MergeResult sliceMerge(SplitResult splitResult, int sliceNum, int origNum) {
EncodeAndDecode encode = new EncodeAndDecode();
List<FileSlice> fileSliceListInput = splitResult.getFileSliceList();
BigInteger[][] encodeDataBigInt = new BigInteger[fileSliceListInput.size()][1]; //从fileSliceList中任取m个数据碎片---7
for (int i = 0; i < fileSliceListInput.size(); i++) {
encodeDataBigInt[i][0] = new BigInteger(fileSliceListInput.get(i).getFileBytes());
}
//encodeDataBigInt转为encodeDataBigDec
BigDecimal[][] encodeDataBigDec = encode.bigIntToDecimal(encodeDataBigInt);
/*从编码矩阵中取对应的行,形成新的encodeMatrixNew*/
BigDecimal[][] encodeMatrix = encode.encodeMatrix(origNum,sliceNum-origNum);
BigDecimal[][] encodeMatrixNew = new BigDecimal[fileSliceListInput.size()][fileSliceListInput.size()];
for (int i = 0; i < encodeMatrixNew.length; i++) {
for (int j = 0; j < encodeMatrixNew[0].length; j++) {
encodeMatrixNew[i][j] = encodeMatrix[fileSliceListInput.get(i).getSliceId()][j];
}
}
/*求新的编码矩阵的逆矩阵*/
Fraction[][] encodeMatrixNewInv = encode.getReverseMartrix(encodeMatrixNew);
/*两矩阵相乘*/
BigDecimal[][] origDataBigDec = encode.encodeData(encodeMatrixNewInv,encodeDataBigDec);
/*BigDecimal转BigInteger*/
BigInteger[] origDataBigInt = encode.bigDecToInt(origDataBigDec);
/*origDataBigInt型数组转为byte数组(origDataBytes)*/
byte[] fileByteDest = encode.bigIntegerToBytes(origDataBigInt,fileSliceListInput.get(0).getSize());
MergeResult mergeResult = new MergeResult();
mergeResult.setFileBytes(fileByteDest);
return mergeResult;
}
@Override
public SendResult sendFileSlice(String targetIp, FileSlice fileSlice) {
return null;
}
@Override
public FileSlice getFileSlice(String targetIp, Long fileId, int sliceId) {
return null;
}
}
package top.ninwoo.app.dfs.util;
import lombok.Builder;
import lombok.Getter;
import java.math.BigDecimal;
@Builder
@Getter
public class Fraction {
private BigDecimal mole;
private BigDecimal deno;
private boolean negate = false;
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>cnf</artifactId>
<groupId>top.ninwoo</groupId>
<version>1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>cnf-distributed-file-transfer</artifactId>
<packaging>pom</packaging>
<modules>
<module>dfs-common-api</module>
<module>dfs-file-utils</module>
<module>dfs-transfer-client</module>
<module>dfs-transfer-server</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>cnf-client-starter</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-transfer-server</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-common-api</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>top.ninwoo</groupId>
<artifactId>dfs-file-utils</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${springboot.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
<version>${lombok.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
</dependency>-->
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
<version>${thymeleaf.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.ujmp/ujmp-core -->
<dependency>
<groupId>org.ujmp</groupId>
<artifactId>ujmp-core</artifactId>
<version>${ujmp.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${springboot.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<target>8</target>
<source>8</source>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
......@@ -20,6 +20,8 @@
<module>apps/cnf-weixingsim</module>
<module>apps/cnf-case-dis</module>
<module>apps/cnf-app-demo</module>
<module>apps/cnf-distributed-file-transfer</module>
<module>apps/cnf-distributed-business-computing</module>
</modules>
......@@ -29,6 +31,9 @@
<spring.version>5.1.4.RELEASE</spring.version>
<docker.version>8.16.0</docker.version>
<curator.version>2.12.0</curator.version>
<lombok.version>1.18.12</lombok.version>
<thymeleaf.version>2.3.0.RELEASE</thymeleaf.version>
<ujmp.version>0.3.0</ujmp.version>
</properties>
<dependencyManagement>
......@@ -127,7 +132,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.10</version>
<version>${lombok.version}</version>
</dependency>
<dependency>
......@@ -155,6 +160,21 @@
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<!--ymwang业务处理中的依赖版本-->
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-thymeleaf -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
<version>${thymeleaf.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.ujmp/ujmp-core -->
<dependency>
<groupId>org.ujmp</groupId>
<artifactId>ujmp-core</artifactId>
<version>${ujmp.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment