diff --git a/pom.xml b/pom.xml index f529a7dbb1112e9ab27d43b62182f7d052cf1877..7ed2930f0a6d1cda8b04e268b58bfe473bd3f716 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ com.xiaotao saltedfishcloud - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE saltedfishcloud 咸鱼云网盘 diff --git a/sfc-api/pom.xml b/sfc-api/pom.xml index 9ed87c287898a85b12f077cb9a912dd4d276ad60..646932b73d8c1e3c9067ef0152e03cb2f08f884d 100644 --- a/sfc-api/pom.xml +++ b/sfc-api/pom.xml @@ -5,7 +5,7 @@ saltedfishcloud com.xiaotao - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE 4.0.0 diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/entity/po/file/FileInfo.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/entity/po/file/FileInfo.java index bfd16f13055c983aecfc4b4bf6b32f988f240b0d..7223428faad7e78771a5649612b21d172afae50b 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/entity/po/file/FileInfo.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/entity/po/file/FileInfo.java @@ -54,7 +54,7 @@ public class FileInfo extends BasicFileInfo{ fileInfo.setName(resource.getFilename()); fileInfo.setUid(uid); fileInfo.setCreatedAt(now); - fileInfo.setSize(resource.contentLength()); + fileInfo.setSize(type == FileInfo.TYPE_DIR ? -1 : resource.contentLength()); fileInfo.setLastModified(now.getTime()); fileInfo.setType(type); fileInfo.setStreamSource(resource); diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/AbstractRawStoreService.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/AbstractRawStoreService.java index 3dd6e24ab7d8e53cbffc08dad002bf51c53c0c6b..ce34a555bffce6f874e4ee3d4ba7646e7211ec8f 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/AbstractRawStoreService.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/AbstractRawStoreService.java @@ -50,6 +50,7 @@ public abstract class AbstractRawStoreService implements StoreService, CustomSto protected FileResourceMd5Resolver md5Resolver; private volatile StoreService uniqueStoreService; + private volatile TempStoreService tempStoreService; public AbstractRawStoreService(DirectRawStoreHandler handler, FileResourceMd5Resolver md5Resolver @@ -319,5 +320,19 @@ public abstract class AbstractRawStoreService implements StoreService, CustomSto copyAndMoveHandler.move(src, dst, overwrite); } - + @Override + public TempStoreService getTempFileHandler() { + // 双重校验锁懒汉单例 + if (tempStoreService != null) { + return tempStoreService; + } + synchronized (this) { + if (tempStoreService != null) { + return tempStoreService; + } + String tempRoot = getTempRoot(); + tempStoreService = new DefaultTempStoreService(handler, tempRoot); + } + return tempStoreService; + } } diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/DefaultTempStoreService.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/DefaultTempStoreService.java new file mode 100644 index 0000000000000000000000000000000000000000..25fe8a6a46aac6c65f00d86a0ef50ece65c1b6c4 --- /dev/null +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/DefaultTempStoreService.java @@ -0,0 +1,100 @@ +package com.xiaotao.saltedfishcloud.service.file; + +import com.xiaotao.saltedfishcloud.entity.po.file.FileInfo; +import com.xiaotao.saltedfishcloud.service.file.store.DirectRawStoreHandler; +import com.xiaotao.saltedfishcloud.utils.StringUtils; +import org.springframework.core.io.Resource; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +/** + * 默认的临时存储服务实现，直接给原始直接存储操作器的路径操作添加临时目录前缀 + */ +public class DefaultTempStoreService implements TempStoreService { + private final DirectRawStoreHandler handler; + private final String tempRootDir; + + public DefaultTempStoreService(DirectRawStoreHandler handler, String tempRootDir) { + this.handler = handler; + this.tempRootDir = tempRootDir; + } + + @Override + public boolean mkdirs(String path) throws IOException { + return handler.mkdirs(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public boolean exist(String path) { + return handler.exist(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public void clean() throws IOException { + handler.delete(tempRootDir); + } + + @Override + public boolean isEmptyDirectory(String path) throws IOException { + return handler.isEmptyDirectory(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public OutputStream newOutputStream(String path) throws IOException { + return handler.newOutputStream(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public Resource getResource(String path) throws IOException { + return handler.getResource(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public List listFiles(String path) throws IOException { + return handler.listFiles(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public FileInfo getFileInfo(String path) throws IOException { + return handler.getFileInfo(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public boolean delete(String path) throws IOException { + return handler.delete(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public boolean mkdir(String path) throws IOException { + return handler.mkdir(StringUtils.appendPath(tempRootDir, path)); + } + + @Override + public long store(String path, InputStream inputStream) throws IOException { + return handler.store(StringUtils.appendPath(tempRootDir, path), inputStream); + } + + @Override + public boolean rename(String path, String newName) throws IOException { + return handler.rename(StringUtils.appendPath(tempRootDir, path), newName); + } + + @Override + public boolean copy(String src, String dest) throws IOException { + return handler.copy( + StringUtils.appendPath(tempRootDir, src), + StringUtils.appendPath(tempRootDir, dest) + ); + } + + @Override + public boolean move(String src, String dest) throws IOException { + return handler.move( + StringUtils.appendPath(tempRootDir, src), + StringUtils.appendPath(tempRootDir, dest) + ); + } +} diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreService.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreService.java index 830e43309adace0d9034f77d727bbb81d2aff0fd..76380dbe0f74f878ee9688e0340e7bc473304097 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreService.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreService.java @@ -4,6 +4,7 @@ import com.xiaotao.saltedfishcloud.exception.JsonException; import com.xiaotao.saltedfishcloud.exception.UnableOverwriteException; import com.xiaotao.saltedfishcloud.entity.po.file.BasicFileInfo; import com.xiaotao.saltedfishcloud.entity.po.file.FileInfo; +import com.xiaotao.saltedfishcloud.service.file.store.DirectRawStoreHandler; import org.springframework.core.io.Resource; import org.springframework.dao.DuplicateKeyException; @@ -28,6 +29,12 @@ public interface StoreService { */ boolean isUnique(); + /** + * 获取一个在目标存储系统的临时目录上，用于以原始路径操作临时文件的文件操作器。 + * @return 临时存储服务 + */ + TempStoreService getTempFileHandler(); + /** * 获取原始存储服务，提供相同文件仅存一份的能力。
* 当当前存储服务为原始存储时，通过该方法获取到的实例应为自身。
diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreServiceProvider.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreServiceProvider.java index 7847cfe199c2a92950fa18b6b900ae1a7dd4a86c..6cf08c02f335a5c093bf0459744bf89ddede229f 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreServiceProvider.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/StoreServiceProvider.java @@ -6,4 +6,8 @@ package com.xiaotao.saltedfishcloud.service.file; */ public interface StoreServiceProvider { StoreService getService(); + + default TempStoreService getTempStoreService() { + return getService().getTempFileHandler(); + } } diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/TempStoreService.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/TempStoreService.java new file mode 100644 index 0000000000000000000000000000000000000000..12fccd3320aba7ac445d7e19b56dad9d3e91ea56 --- /dev/null +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/TempStoreService.java @@ -0,0 +1,16 @@ +package com.xiaotao.saltedfishcloud.service.file; + +import com.xiaotao.saltedfishcloud.service.file.store.DirectRawStoreHandler; + +import java.io.IOException; + +/** + * 在目标存储服务上以临时文件目录为根目录，提供临时文件操作功能。 + */ +public interface TempStoreService extends DirectRawStoreHandler { + + /** + * 清空临时目录 + */ + void clean() throws IOException; +} diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/DirectRawStoreHandler.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/DirectRawStoreHandler.java index 1e5a3b1b8102784db24e65417de71db91c9170af..d141cda5e9b29c1f5a00f3e9f232b189cc2c4460 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/DirectRawStoreHandler.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/DirectRawStoreHandler.java @@ -4,13 +4,14 @@ import com.xiaotao.saltedfishcloud.entity.po.file.FileInfo; import com.xiaotao.saltedfishcloud.service.file.store.StoreReader; import com.xiaotao.saltedfishcloud.service.file.store.StoreWriter; import com.xiaotao.saltedfishcloud.utils.PathUtils; +import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.Resource; import java.io.IOException; import java.io.InputStream; /** - * 直接使用存储服务的原始文件路径（而不是网盘路径）进行资源操作的服务，提供目标文件存储系统最基础最原始的文件操作。 + * 直接使用存储服务的原始文件路径（而不是网盘路径）进行资源操作的服务，提供目标文件存储系统最基础最原始的文件操作。
* 通常不直接使用，而是为其他涉及到文件存储操作的抽象类提供文件最基本的存储能力（如抽象文件存储服务，抽象用户配置数据服务）。 */ public interface DirectRawStoreHandler extends StoreReader, StoreWriter { @@ -22,12 +23,19 @@ public interface DirectRawStoreHandler extends StoreReader, StoreWriter { */ @Override default boolean mkdirs(String path) throws IOException { + final FileInfo curPathInfo = getFileInfo(path); if (curPathInfo == null) { final String parentPath = PathUtils.getParentPath(path); final FileInfo parentPathInfo = getFileInfo(parentPath); - if (parentPathInfo != null && parentPathInfo.isDir()) { - return mkdir(parentPath); + + if (parentPathInfo == null) { + mkdirs(parentPath); + return mkdir(path); + } + + if (parentPathInfo.isDir()) { + return mkdir(path); } else { return false; } diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/StoreWriter.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/StoreWriter.java index 20f0c02c544ecdfb8b6281c20f4be85b42e39058..6d9b8dbc78ec660cafa274b17935f59cb93f7054 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/StoreWriter.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/service/file/store/StoreWriter.java @@ -2,6 +2,7 @@ package com.xiaotao.saltedfishcloud.service.file.store; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; /** * 提供存储系统的资源创建与删除能力 @@ -28,6 +29,14 @@ public interface StoreWriter { */ long store(String path, InputStream inputStream) throws IOException; + /** + * 获取目标路径文件资源的输出流 + * @param path 文件保存路径 + * @return 输出流，往流中写入的数据将写入存储系统 + * @throws IOException 任意IO错误 + */ + OutputStream newOutputStream(String path) throws IOException; + /** * 对文件资源进行重命名 * @param path 文件完整路径 diff --git a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/utils/PathUtils.java b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/utils/PathUtils.java index 2157d2f197a4533d3a0f15b92463a18931903bb3..1f7a71cc61799ab06549fc45114a8bf8409aaef8 100644 --- a/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/utils/PathUtils.java +++ b/sfc-api/src/main/java/com/xiaotao/saltedfishcloud/utils/PathUtils.java @@ -18,6 +18,14 @@ public class PathUtils { * @return 父级路径 */ public static String getParentPath(String path) { + if (path.endsWith("/")) { + path = path.replaceAll("/+$", ""); + + // 类似只有/的路径被正则替换后，无了 + if (path.length() == 0) { + return "/"; + } + } final int i = path.lastIndexOf('/'); if (i == 0) { return "/"; @@ -34,6 +42,9 @@ public class PathUtils { * @return 节点名称 */ public static String getLastNode(String path) { + if (path.endsWith("/")) { + path = path.replaceAll("/+$", ""); + } int pos = path.lastIndexOf("/"); if (pos == -1) { return path; diff --git a/sfc-compress/pom.xml b/sfc-compress/pom.xml index 3ab5a570800b1935885cf8b146a42d570200c381..732497175440833f8b99266494f4abad85a11720 100644 --- a/sfc-compress/pom.xml +++ b/sfc-compress/pom.xml @@ -7,7 +7,7 @@ com.xiaotao saltedfishcloud - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE sfc-compress @@ -15,7 +15,7 @@ com.xiaotao sfc-api - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE provided diff --git a/sfc-core/pom.xml b/sfc-core/pom.xml index cc7c52245a1e7c96cc65f381609d2c7719951584..60022dd1bac2aec0437c1f6b5027ca94c7f33a80 100644 --- a/sfc-core/pom.xml +++ b/sfc-core/pom.xml @@ -5,7 +5,7 @@ saltedfishcloud com.xiaotao - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE 4.0.0 @@ -20,7 +20,7 @@ com.xiaotao sfc-api - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE @@ -70,7 +70,7 @@ com.xiaotao sfc-compress - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE compile diff --git a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/controller/FileController.java b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/controller/FileController.java index 59f365795ad5b3f588c645ed4786b022cd35cf59..032093dbae295ddfd4f4239ed72648c618769c3b 100644 --- a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/controller/FileController.java +++ b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/controller/FileController.java @@ -23,11 +23,13 @@ import com.xiaotao.saltedfishcloud.service.file.DiskFileSystemProvider; import com.xiaotao.saltedfishcloud.service.wrap.WrapInfo; import com.xiaotao.saltedfishcloud.service.wrap.WrapService; import com.xiaotao.saltedfishcloud.utils.FileUtils; +import com.xiaotao.saltedfishcloud.utils.PathUtils; import com.xiaotao.saltedfishcloud.utils.ResourceUtils; import com.xiaotao.saltedfishcloud.utils.URLUtils; import com.xiaotao.saltedfishcloud.validator.annotations.FileName; import com.xiaotao.saltedfishcloud.validator.annotations.UID; import lombok.RequiredArgsConstructor; +import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -216,18 +218,21 @@ public class FileController { /** * 获取网盘文件内容（文件下载） + * @deprecated 该接口将启用，文件下载请使用{@link ResourceController#downloadByMD5(String, int, HttpServletRequest)}替代 */ @RequestMapping(value = "content/**", method = {RequestMethod.POST, RequestMethod.GET}) @AllowAnonymous @NotBlock(level = ProtectLevel.DATA_CHECKING) - public ResponseEntity download(HttpServletRequest request, - @PathVariable @UID int uid) + @Deprecated + public ResponseEntity download(HttpServletRequest request,@PathVariable @UID int uid) throws IOException { String prefix = PREFIX + uid + "/content"; String requestPath = URLUtils.getRequestFilePath(prefix, request); - org.springframework.core.io.Resource resource = fileService.getFileSystem().getResource(uid, requestPath, ""); + String dir = PathUtils.getParentPath(requestPath); + String name = PathUtils.getLastNode(requestPath); + Resource resource = fileService.getFileSystem().getResource(uid, dir, name); if (resource != null) { - return ResourceUtils.wrapResource(resource); + return ResourceUtils.wrapResource(resource, name); } else { throw new JsonException(FileSystemError.FILE_NOT_FOUND); } diff --git a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/DefaultTaskManager.java b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/DefaultTaskManager.java index bdb16987110763c03efcb13c973484926bd4f9e5..a8069617d968171303757d544aaacd66b1efa687 100644 --- a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/DefaultTaskManager.java +++ b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/DefaultTaskManager.java @@ -7,18 +7,23 @@ import com.xiaotao.saltedfishcloud.service.breakpoint.manager.TaskManager; import com.xiaotao.saltedfishcloud.service.breakpoint.manager.impl.utils.TaskStorePath; import com.xiaotao.saltedfishcloud.service.breakpoint.merge.MergeInputStream; import com.xiaotao.saltedfishcloud.service.breakpoint.merge.MultipleFileMergeInputStreamGenerator; +import com.xiaotao.saltedfishcloud.service.file.StoreServiceProvider; +import com.xiaotao.saltedfishcloud.service.file.TempStoreService; import com.xiaotao.saltedfishcloud.utils.FileUtils; -import com.xiaotao.saltedfishcloud.utils.MapperHolder; import lombok.extern.slf4j.Slf4j; import lombok.var; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.util.StreamUtils; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; -import java.util.UUID; +import java.time.Duration; +import java.util.*; import java.util.stream.Collectors; /** @@ -27,6 +32,19 @@ import java.util.stream.Collectors; @Slf4j public class DefaultTaskManager implements TaskManager { + @Autowired + private RedisTemplate redisTemplate; + + @Autowired + private StoreServiceProvider storeServiceProvider; + + private String getMetaRedisKey(String id) { + return "xyy::breakpoint::" + id; + } + + private String getFinishPartKey(String id) { + return "xyy::breakpoint::finish::" + id; + } /** * 创建断点续传任务 @@ -39,9 +57,8 @@ public class DefaultTaskManager implements TaskManager { var id = UUID.randomUUID().toString(); info.setTaskId(id); var taskDir = TaskStorePath.getRoot(id); - Files.createDirectories(taskDir); - Files.write(TaskStorePath.getMetadata(id), MapperHolder.mapper.writeValueAsBytes(info)); - + storeServiceProvider.getTempStoreService().mkdirs(taskDir); + redisTemplate.opsForValue().set(getMetaRedisKey(id), info, Duration.ofDays(7)); return id; } @@ -53,12 +70,8 @@ public class DefaultTaskManager implements TaskManager { */ @Override public TaskMetadata queryTask(String id) throws IOException { - var metadataPath = TaskStorePath.getMetadata(id); - if (!Files.exists(metadataPath)) { - throw new TaskNotFoundException(id); - } - return MapperHolder.mapper.readValue(Files.readAllBytes(metadataPath), TaskMetadata.class); + return (TaskMetadata) redisTemplate.opsForValue().get(getMetaRedisKey(id)); } /** @@ -68,11 +81,13 @@ public class DefaultTaskManager implements TaskManager { */ @Override public void clear(String id) throws IOException { - var taskPath = TaskStorePath.getRoot(id); - if (!Files.exists(taskPath)) { + if(queryTask(id) == null) { throw new TaskNotFoundException(id); } - FileUtils.delete(taskPath); + var taskPath = TaskStorePath.getRoot(id); + storeServiceProvider.getTempStoreService().delete(taskPath); + redisTemplate.delete(getMetaRedisKey(id)); + redisTemplate.delete(getFinishPartKey(id)); } /** @@ -84,28 +99,31 @@ public class DefaultTaskManager implements TaskManager { @Override public void save(String id, String part, InputStream stream) throws IOException { var root = TaskStorePath.getRoot(id); - if (!Files.exists(root)) { + if (queryTask(id) == null) { throw new TaskNotFoundException(id); } + final String redisKey = getFinishPartKey(id); var parts = PartParser.parse(part); var taskInfo = queryTask(id); for (int i : parts) { var size = taskInfo.getPartSize(i); - var out = Files.newOutputStream(TaskStorePath.getPartFile(id, i)); - long l = StreamUtils.copyRange(stream, out, 0, size - 1); - log.debug("写入断点续传文件块，文件名：{} 编号：{} 大小：{}",taskInfo.getFileName() ,i, l); - out.close(); + final String partFile = TaskStorePath.getPartFile(id, i); + try(final OutputStream out = storeServiceProvider.getTempStoreService().newOutputStream(partFile)) { + long l = StreamUtils.copyRange(stream, out, 0, size - 1); + log.debug("写入断点续传文件块，文件名：{} 编号：{} 大小：{}",taskInfo.getFileName() ,i, l); + redisTemplate.opsForSet().add(redisKey, i); + } } stream.close(); } @Override public List getFinishPart(String id) throws IOException { - return Files.list(TaskStorePath.getRoot(id)) - .filter(e -> e.toString().endsWith(".part")) - .map(e -> Integer.parseInt(e.getFileName().toString().replaceAll(".part", ""))) - .sorted() - .collect(Collectors.toList()); + Set finish = redisTemplate.opsForSet().members(getFinishPartKey(id)); + if (finish == null) { + return Collections.emptyList(); + } + return finish.stream().map(e -> (Integer)e).sorted(Comparator.comparingInt(o -> o)).collect(Collectors.toList()); } @Override @@ -129,9 +147,10 @@ public class DefaultTaskManager implements TaskManager { List finishPart = getFinishPart(id); - Path[] paths = new Path[finishPart.size()]; + Resource[] paths = new Resource[finishPart.size()]; + final TempStoreService tempStoreService = storeServiceProvider.getTempStoreService(); for (Integer integer : finishPart) { - paths[integer - 1] = TaskStorePath.getPartFile(id, integer); + paths[integer - 1] = tempStoreService.getResource(TaskStorePath.getPartFile(id, integer)); } return new MergeInputStream(new MultipleFileMergeInputStreamGenerator(paths)); } diff --git a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/utils/TaskStorePath.java b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/utils/TaskStorePath.java index d1a462b8ea39bed787def5ab62e2fe401ff1e0a9..c0045bf6eaa0f5015e0273513e0bee19a6cba208 100644 --- a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/utils/TaskStorePath.java +++ b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/manager/impl/utils/TaskStorePath.java @@ -10,11 +10,11 @@ public class TaskStorePath { * 获取任务数据文件夹路径 * @param id 任务ID */ - public static Path getRoot(String id) { + public static String getRoot(String id) { if (id == null) { throw new NullPointerException(); } - return Paths.get(PathUtils.getTempDirectory() + "/xyy/" + id); + return "/breakpoint/" + id; } /** @@ -22,15 +22,9 @@ public class TaskStorePath { * @param id 任务ID * @param part 文件块 */ - public static Path getPartFile(String id, int part) { - return Paths.get(getRoot(id) + "/" + part + ".part"); + public static String getPartFile(String id, int part) { + return getRoot(id) + "/" + part + ".part"; } - /** - * 获取任务元数据路径 - * @param id 任务ID - */ - public static Path getMetadata(String id) { - return Paths.get(getRoot(id) + "/metadata.json"); - } + } diff --git a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/merge/MultipleFileMergeInputStreamGenerator.java b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/merge/MultipleFileMergeInputStreamGenerator.java index 846aa20b2be74a18fab3fa461a49b8c5191a4e5e..849720b9c91f320c1ecc7e4b53c05e3b99c6c85a 100644 --- a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/merge/MultipleFileMergeInputStreamGenerator.java +++ b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/breakpoint/merge/MultipleFileMergeInputStreamGenerator.java @@ -1,25 +1,27 @@ package com.xiaotao.saltedfishcloud.service.breakpoint.merge; +import org.springframework.core.io.InputStreamSource; + import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; public class MultipleFileMergeInputStreamGenerator implements InputStreamGenerator { - private final Path[] paths; + private final InputStreamSource[] sources; private int index = 0; - public MultipleFileMergeInputStreamGenerator(Path...paths) { - this.paths = paths; + public MultipleFileMergeInputStreamGenerator(InputStreamSource...sources) { + this.sources = sources; } @Override public InputStream next() throws IOException { if (!hasNext()) return null; - return Files.newInputStream(paths[index++]); + return sources[index++].getInputStream(); } @Override public boolean hasNext() { - return index < paths.length; + return index < sources.length; } } diff --git a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/file/impl/store/LocalDirectRawStoreHandler.java b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/file/impl/store/LocalDirectRawStoreHandler.java index 34d08400b0fc9eee20751baf116fe04f75aacea1..9de681e397b729331fe2215a410570cdb22cf830 100644 --- a/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/file/impl/store/LocalDirectRawStoreHandler.java +++ b/sfc-core/src/main/java/com/xiaotao/saltedfishcloud/service/file/impl/store/LocalDirectRawStoreHandler.java @@ -26,6 +26,22 @@ import java.util.List; */ @Slf4j public class LocalDirectRawStoreHandler implements DirectRawStoreHandler { + @Override + public OutputStream newOutputStream(String path) throws IOException { + if (exist(path)) { + if (getFileInfo(path).isDir()) { + throw new UnsupportedOperationException("不支持向文件夹写入数据"); + } else { + delete(path); + } + } else { + final String parent = PathUtils.getParentPath(path); + mkdirs(parent); + } + + return Files.newOutputStream(Paths.get(path)); + } + @Override public Resource getResource(String path) throws IOException { final Path localPath = Paths.get(path); @@ -60,7 +76,11 @@ public class LocalDirectRawStoreHandler implements DirectRawStoreHandler { @Override public FileInfo getFileInfo(String path) throws IOException { - return FileInfo.getLocal(path, false); + final Path path1 = Paths.get(path); + if (!Files.exists(path1)) { + return null; + } + return FileInfo.getFromResource(new PathResource(path1), 0, Files.isDirectory(path1) ? FileInfo.TYPE_DIR : FileInfo.TYPE_FILE); } @Override diff --git a/sfc-core/src/main/resources/application.yml b/sfc-core/src/main/resources/application.yml index 044af9b94cdf441ddf5eb4a95ba3f4bae58c65a0..b7103486e94bdef8a0dfe2ccf80e415dce96a1fc 100644 --- a/sfc-core/src/main/resources/application.yml +++ b/sfc-core/src/main/resources/application.yml @@ -18,9 +18,9 @@ spring: type: com.alibaba.druid.pool.DruidDataSource druid: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1/t_xyy?useSSL=false&serverTimezone=UTC + url: jdbc:mysql://127.0.0.1/d_xyy?useSSL=false&serverTimezone=UTC username: root - password: + password: mojintao233 test-while-idle: true test-on-borrow: false test-on-return: false @@ -41,8 +41,8 @@ logging: level: org: warn com: warn - com.xiaotao: info -# com.xiaotao.saltedfishcloud.dao.mybatis: warn + com.xiaotao: debug + com.xiaotao.saltedfishcloud.dao.mybatis: warn com.xiaotao.saltedfishcloud.SaltedfishcloudApplication: warn app: version: ^project.version^ diff --git a/sfc-ext/sfc-ext-hadoop-store/src/main/java/com/saltedfishcloud/ext/hadoop/store/HDFSStoreHandler.java b/sfc-ext/sfc-ext-hadoop-store/src/main/java/com/saltedfishcloud/ext/hadoop/store/HDFSStoreHandler.java index 217e452775d2e6ec866320cc5916f02cd47310ff..45c6b2a1a24a89123f9041fa4bf1e9b7e225ebf9 100644 --- a/sfc-ext/sfc-ext-hadoop-store/src/main/java/com/saltedfishcloud/ext/hadoop/store/HDFSStoreHandler.java +++ b/sfc-ext/sfc-ext-hadoop-store/src/main/java/com/saltedfishcloud/ext/hadoop/store/HDFSStoreHandler.java @@ -13,6 +13,7 @@ import org.springframework.util.StreamUtils; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; public class HDFSStoreHandler extends HDFSReader implements DirectRawStoreHandler { private final FileSystem fs; @@ -21,6 +22,18 @@ public class HDFSStoreHandler extends HDFSReader implements DirectRawStoreHandle this.fs = fs; } + @Override + public OutputStream newOutputStream(String path) throws IOException { + Path target = new Path(path); + if (fs.exists(target)) { + if (fs.getFileStatus(target).isDirectory()) { + throw new JsonException(FileSystemError.RESOURCE_TYPE_NOT_MATCH); + } + fs.delete(target, false); + } + return fs.create(target); + } + @Override public boolean delete(String path) throws IOException { return fs.delete(new Path(path), true); diff --git a/sfc-orm-configure/pom.xml b/sfc-orm-configure/pom.xml index ae3e603c8d5820669a6661de4fc1a5c59029b7d3..c9aed309b44c8e12cc93ea0f01980c57d8398d8d 100644 --- a/sfc-orm-configure/pom.xml +++ b/sfc-orm-configure/pom.xml @@ -5,7 +5,7 @@ saltedfishcloud com.xiaotao - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE 4.0.0 diff --git a/sfc-test-module/pom.xml b/sfc-test-module/pom.xml index 0aef281952ea716720ab6f11f873e7466723606d..7c35b12e861e4ae6ead0f96c098cedeb45fd678c 100644 --- a/sfc-test-module/pom.xml +++ b/sfc-test-module/pom.xml @@ -5,7 +5,7 @@ saltedfishcloud com.xiaotao - 1.8.4.2-RELEASE + 1.8.4.3-RELEASE 4.0.0