0727 动态数据集 完成前序文件删除 并且数据保存至mino 中，本地不存储数据。数据集的下载链接已更新。

src/main/java/io/renren/modules/dataSet/service/impl/DynamicSystemServiceImpl.java

@@ -1,5 +1,6 @@
 package io.renren.modules.dataSet.service.impl;
 
+import cn.hutool.core.io.FileUtil;
 import com.alibaba.fastjson.JSON;
 import io.minio.MinioClient;
 import io.minio.PutObjectOptions;
@@ -283,6 +284,10 @@ public class DynamicSystemServiceImpl implements DynamicSystemService {
         final String userID =  map.get("userID").toString();
         // path 文件保存路径 需要改四次  本类三次  StaticSystemServiceImpl 一次
         String path  = System.getProperty("user.dir")+ File.separator+"csv" + File.separator + userID;  // 文件路径  记得后面的dySubmit也要改
+        // 判断上传路径是否存在
+        if (!FileUtil.exist(path)){
+            FileUtil.mkdir(path);
+        }
         String filed = new String();
         List<String> fileds = new ArrayList<>();
 
@@ -455,7 +460,9 @@ public class DynamicSystemServiceImpl implements DynamicSystemService {
             }
             try {
                 PutObjectOptions putObjectOptions = new PutObjectOptions(newFileName.length(), PutObjectOptions.MIN_MULTIPART_SIZE);
-                MinIoUtils.uploadLocalFile("dataset",newFileName.getName(),newFileName.getAbsolutePath(),putObjectOptions);
+                MinIoUtils.uploadLocalFile("dydataset",newFileName.getName(),newFileName.getAbsolutePath(),putObjectOptions);
+                //  删除本地数据文件
+                FileUtil.del(newFileName);
             }catch (Exception e){
                 System.out.println(e);
             }

src/main/java/io/renren/modules/dataSet/service/impl/StaticSystemServiceImpl.java

@@ -1,5 +1,6 @@
 package io.renren.modules.dataSet.service.impl;
 
+import cn.hutool.core.io.FileUtil;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.fastjson.JSON;
 import io.minio.PutObjectOptions;
@@ -41,8 +42,10 @@ public class StaticSystemServiceImpl implements StaticSystemService {
         final String lastNewFileName = onemap1.get("newFileName").toString();
         final String type = onemap1.get("type").toString();
         String path  = System.getProperty("user.dir")+ File.separator+"csv" + File.separator + userID;
-
-
+        // 判断上传路径是否存在
+        if (!FileUtil.exist(path)){
+            FileUtil.mkdir(path);
+        }
 
         List<Map<String, Object>> allList = null;
         if ("excel".equals(type)) {
@@ -133,6 +136,8 @@ public class StaticSystemServiceImpl implements StaticSystemService {
         try {
             PutObjectOptions putObjectOptions = new PutObjectOptions(newFileName.length(), PutObjectOptions.MIN_MULTIPART_SIZE);
             MinIoUtils.uploadLocalFile("dataset",newFileName.getName(),newFileName.getAbsolutePath(),putObjectOptions);
+            //  删除本地数据文件
+            FileUtil.del(newFileName);
         }catch (Exception e){
             System.out.println(e);
         }

src/main/java/io/renren/modules/dataSet/upload/UpLoadService.java

@@ -1,6 +1,7 @@
 package io.renren.modules.dataSet.upload;
 
 
+import cn.hutool.core.io.FileUtil;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.fastjson.JSON;
 import io.renren.common.exception.RRException;
@@ -21,8 +22,6 @@ import java.util.stream.Collectors;
 public class UpLoadService {
     public Map upload(MultipartFile file) throws IOException {
         Map<String, Object> map = new HashMap<>();
-
-
         if (file.isEmpty()) {
             throw new RRException("上传文件不能为空");
         }
@@ -138,12 +137,16 @@ public class UpLoadService {
         //获取上传的文件的文件名
         String fileName = file.getOriginalFilename();
         map.put("fileName", fileName);
-        String newfileName = System.getProperty("user.dir") + File.separator + "aupload" + File.separator + fileName;
+        String dirtory = System.getProperty("user.dir") + File.separator + "aupload";
+
+        // 判断上传路径是否存在
+        if (!FileUtil.exist(dirtory)){
+            FileUtil.mkdir(dirtory);
+        }
+        String newfileName = dirtory+ File.separator + fileName;
         File newfile = new File(newfileName);
         file.transferTo(newfile);
         map.put("upUrl", newfileName);
-
-
         return readWithCsvMapReader(newfileName, map, false);
     }
 

src/main/java/io/renren/modules/job/controller/ScheduleJobController.java

@@ -8,7 +8,9 @@
 
 package io.renren.modules.job.controller;
 
+import io.minio.errors.*;
 import io.renren.common.annotation.SysLog;
+import io.renren.common.utils.MinIoUtils;
 import io.renren.common.utils.PageUtils;
 import io.renren.common.utils.R;
 import io.renren.common.validator.ValidatorUtils;
@@ -22,6 +24,9 @@ import org.checkerframework.checker.units.qual.A;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.Map;
 
 /**
@@ -137,7 +142,13 @@ public class ScheduleJobController {
 		for(Long datasetId:datasetIds){
 			System.out.println(datasetId);
 			Long jobId = dataSetDyService.getById(datasetId).getJobId();
+			DataSetDy dataSetDy = dataSetDyService.getById(datasetId);
 			dataSetDyService.removeById(datasetId);
+			try {
+				MinIoUtils.deleteFile("dydataset",dataSetDy.getDatasetName()+".csv");
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
 			i = i+1;
 			jobIds[i] = jobId;
 		}

src/main/java/io/renren/modules/job/task/demo1.java

@@ -1,7 +1,11 @@
 package io.renren.modules.job.task;
 
 
+import cn.hutool.core.io.FileUtil;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import io.minio.PutObjectOptions;
+import io.minio.errors.*;
+import io.renren.common.utils.MinIoUtils;
 import io.renren.modules.dataSet.entity.DataSetDychoosedata;
 import io.renren.modules.dataSet.mapper.DataSetDychoosedataMapper;
 import io.renren.modules.dataSet.service.DynamicSystemService;
@@ -12,6 +16,10 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.io.File;
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.*;
 
 @Component("demo1")
@@ -27,8 +35,8 @@ public class demo1 implements ITask{
     private DynamicSystemService dynamicSystemService;
     @Override
     public void run(String params) {
-        System.out.println("demo1 执行了  时间------" + new Date());
-        System.out.println(params);
+//        System.out.println("demo1 执行了  时间------" + new Date());
+//        System.out.println(params);
         logger.debug("TestTask定时任务正在执行，参数为：{}", params);
         Integer parms = Integer.parseInt(params);
 
@@ -89,6 +97,21 @@ public class demo1 implements ITask{
 
             }
             // 最后 更新数据集中 文件路径
+
+            String path  = System.getProperty("user.dir")+ File.separator+"csv" + File.separator + map.get("userID");
+            File newFileName = new File(path,map.get("newFileName")+ ".csv");
+            File lastFileName = new File(path,dataSetDy.getDatasetName()+".csv");
+            FileUtil.rename(newFileName,lastFileName.getName(),false,true);
+            PutObjectOptions putObjectOptions = new PutObjectOptions(lastFileName.length(), PutObjectOptions.MIN_MULTIPART_SIZE);
+            try {
+                MinIoUtils.uploadLocalFile("dydataset",lastFileName.getName(),lastFileName.getAbsolutePath(),putObjectOptions);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            //  删除本地数据文件
+            FileUtil.del(lastFileName);
+
+
             dataSetDy.setDfsUrl(map.get("newFileName").toString());
             dataSetDyMapper.updateById(dataSetDy);
         }

src/main/java/io/renren/modules/sys/controller/dataset/DataSetDyController.java

@@ -2,6 +2,7 @@ package io.renren.modules.sys.controller.dataset;
 
 
 import io.renren.common.utils.Constant;
+import io.renren.common.utils.MinIoUtils;
 import io.renren.common.utils.PageUtils;
 import io.renren.common.utils.R;
 import io.renren.modules.sys.service.DataSetDyService;
@@ -40,8 +41,18 @@ public class DataSetDyController {
         }
         PageUtils page = dataSetDyService.queryPageOnUse(params);
 
-
         return R.ok().put("page", page);
     }
+    /**
+     * Description 下载数据集
+     * @param datasetName
+     * @return
+     */
+    @RequestMapping("/downloadDataset")
+    public R downloadStaticDataset(String datasetName){
+        //从minio获得数据集的url，直接返回给前端
+        String downloadUrl= MinIoUtils.getFileUrl("dydataset",datasetName);
+        return R.ok().put("downloadUrl",downloadUrl);
+    }
 }
 

src/main/resources/application-dev.yml

@@ -5,8 +5,8 @@ spring:
             driver-class-name: com.mysql.cj.jdbc.Driver
             url: jdbc:mysql://localhost:3306/renren_fast?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
             username: root
-#            password: 123456
-            password: root
+            password: 123456
+#            password: root
             initial-size: 10
             max-active: 100
             min-idle: 10
@@ -44,8 +44,8 @@ dynamic:
         url: jdbc:mysql://localhost:3306/hebing?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
         username: root
 #      password: XDUbdm520Mysql
-        password: root
-#        password: 123456
+#        password: root
+        password: 123456
         driver-class-name: com.mysql.jdbc.Driver # 3.2.0开始支持SPI可省略此配置
 server:
     port: 8082