Skip to content

Develop/dataset link #430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MoeexT wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Develop/dataset link #430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b4d71c7
fix: file count error after deletion
MoeexT Mar 13, 2026
1b6792d
fix: delete dataset tooltip
MoeexT Mar 13, 2026
3ba0b17
fix: unify deletion pop-up text
MoeexT Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 57 additions & 32 deletions .../src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,34 +226,50 @@ public DatasetFile getDatasetFile(Dataset dataset, String fileId, String prefix)

/**
* 删除文件
* 使用悲观锁防止并发删除导致 fileCount 计算不准确
*/
@Transactional
public void deleteDatasetFile(String datasetId, String fileId, String prefix) {
Dataset dataset = datasetRepository.getById(datasetId);
// 使用悲观锁获取 dataset，确保并发安全
Dataset dataset = datasetRepository.getByIdWithLock(datasetId);
DatasetFile file = getDatasetFile(dataset, fileId, prefix);
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));

// 先删除数据库记录
datasetFileRepository.removeById(fileId);
if (CommonUtils.isUUID(fileId)) {
dataset.removeFile(file);
}
datasetRepository.updateById(dataset);
// 删除文件时，上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件，归集过来的文件仅删除数据库中的记录

// 删除物理文件（仅删除在数据集路径下的文件）
if (file.getFilePath().startsWith(dataset.getPath())) {
try {
Path filePath = validateAndResolvePath(file.getFilePath(), dataset.getPath());
Files.deleteIfExists(filePath);
} catch (IOException ex) {
log.error("删除物理文件失败: {}", file.getFilePath(), ex);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
}

// 重新计算文件统计信息（确保数据准确）
List<DatasetFile> remainingFiles = datasetFileRepository.findAllByDatasetId(datasetId);
dataset.setFileCount((long) remainingFiles.size());
dataset.setSizeBytes(remainingFiles.stream()
.mapToLong(f -> f.getFileSize() != null ? f.getFileSize().longValue() : 0L)
.sum());
dataset.setUpdatedAt(LocalDateTime.now());

datasetRepository.updateById(dataset);

log.info("删除文件成功 - datasetId: {}, fileId: {}, fileName: {}, fileCount: {}, sizeBytes: {}",
datasetId, fileId, file.getFileName(), dataset.getFileCount(), dataset.getSizeBytes());
}

/**
* 批量删除文件
* 使用悲观锁防止并发删除导致 fileCount 计算不准确
*/
@Transactional
public void batchDeleteFiles(String datasetId, BatchDeleteFilesRequest request) {
Dataset dataset = datasetRepository.getById(datasetId);
// 使用悲观锁获取 dataset，确保并发安全
Dataset dataset = datasetRepository.getByIdWithLock(datasetId);
if (dataset == null) {
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
}
Expand All @@ -266,6 +282,7 @@ public void batchDeleteFiles(String datasetId, BatchDeleteFilesRequest request)
List<DatasetFile> filesToDelete = new ArrayList<>();
List<String> failedFileIds = new ArrayList<>();

// 先删除数据库记录
for (String fileId : fileIds) {
try {
DatasetFile file = getDatasetFile(dataset, fileId, request.getPrefix());
Expand All @@ -277,20 +294,8 @@ public void batchDeleteFiles(String datasetId, BatchDeleteFilesRequest request)
}
}

// 更新数据集（避免 ConcurrentModificationException）
List<DatasetFile> datasetFiles = dataset.getFiles();
if (datasetFiles != null) {
// 创建一个新的列表来存储要保留的文件
List<DatasetFile> remainingFiles = new ArrayList<>(datasetFiles);
// 移除要删除的文件
remainingFiles.removeAll(filesToDelete);
dataset.setFiles(remainingFiles);
}
datasetRepository.updateById(dataset);

// 删除文件系统中的文件
// 删除物理文件（仅删除在数据集路径下的文件）
for (DatasetFile file : filesToDelete) {
// 上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件，归集过来的文件仅删除数据库中的记录
if (file.getFilePath().startsWith(dataset.getPath())) {
try {
Path filePath = validateAndResolvePath(file.getFilePath(), dataset.getPath());
Expand All @@ -303,6 +308,19 @@ public void batchDeleteFiles(String datasetId, BatchDeleteFilesRequest request)
}
}

// 重新计算文件统计信息（确保数据准确）
List<DatasetFile> remainingFiles = datasetFileRepository.findAllByDatasetId(datasetId);
dataset.setFileCount((long) remainingFiles.size());
dataset.setSizeBytes(remainingFiles.stream()
.mapToLong(f -> f.getFileSize() != null ? f.getFileSize().longValue() : 0L)
.sum());
dataset.setUpdatedAt(LocalDateTime.now());

datasetRepository.updateById(dataset);

log.info("批量删除文件成功 - datasetId: {}, successCount: {}, failedCount: {}, fileCount: {}, sizeBytes: {}",
datasetId, filesToDelete.size(), failedFileIds.size(), dataset.getFileCount(), dataset.getSizeBytes());

// 如果有失败的文件，记录日志但不抛出异常
if (!failedFileIds.isEmpty()) {
log.warn("Failed to delete {} files out of {}", failedFileIds.size(), fileIds.size());
Expand Down Expand Up @@ -643,7 +661,8 @@ private void zipDirectory(Path sourceDir, Path basePath, ZipArchiveOutputStream
*/
@Transactional
public void deleteDirectory(String datasetId, String prefix) {
Dataset dataset = datasetRepository.getById(datasetId);
// 使用悲观锁获取 dataset，防止并发删除导致 fileCount 不准确
Dataset dataset = datasetRepository.getByIdWithLock(datasetId);
if (dataset == null) {
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
}
Expand Down Expand Up @@ -694,6 +713,10 @@ public void deleteDirectory(String datasetId, String prefix) {
})
.collect(Collectors.toList());

log.info("删除目录开始 - datasetId: {}, prefix: {}, fileCount: {}, filesToDelete: {}",
datasetId, prefix, dataset.getFileCount(), filesToDelete.size());

// 删除数据库记录
for (DatasetFile file : filesToDelete) {
datasetFileRepository.removeById(file.getId());
}
Expand All @@ -702,20 +725,22 @@ public void deleteDirectory(String datasetId, String prefix) {
try {
deleteDirectoryRecursively(normalized);
} catch (IOException e) {
log.error("Failed to delete directory {} for dataset {}", normalized, datasetId, e);
log.error("删除目录失败: datasetId={}, prefix={}", datasetId, prefix, e);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}

// 更新数据集（避免 ConcurrentModificationException，先获取文件列表再删除）
List<DatasetFile> datasetFiles = dataset.getFiles();
if (datasetFiles != null) {
// 创建一个新的列表来存储要保留的文件
List<DatasetFile> remainingFiles = new ArrayList<>(datasetFiles);
// 移除要删除的文件
remainingFiles.removeAll(filesToDelete);
dataset.setFiles(remainingFiles);
}
// 重新计算文件统计信息（确保数据准确）
List<DatasetFile> remainingFiles = datasetFileRepository.findAllByDatasetId(datasetId);
dataset.setFileCount((long) remainingFiles.size());
dataset.setSizeBytes(remainingFiles.stream()
.mapToLong(f -> f.getFileSize() != null ? f.getFileSize().longValue() : 0L)
.sum());
dataset.setUpdatedAt(LocalDateTime.now());

datasetRepository.updateById(dataset);

log.info("删除目录成功 - datasetId: {}, prefix: {}, deletedCount: {}, fileCount: {}, sizeBytes: {}",
datasetId, prefix, filesToDelete.size(), dataset.getFileCount(), dataset.getSizeBytes());
}

/**
Expand Down
7 changes: 7 additions & 0 deletions ...ain/java/com/datamate/datamanagement/infrastructure/persistence/mapper/DatasetMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,20 @@
import com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.session.RowBounds;

import java.util.List;

@Mapper
public interface DatasetMapper extends BaseMapper<Dataset> {
Dataset findById(@Param("id") String id);

/**
* 使用悲观锁查询数据集（FOR UPDATE）
*/
@Select("SELECT * FROM t_dm_datasets WHERE id = #{id} FOR UPDATE")
Dataset findByIdWithLock(@Param("id") String id);
Dataset findByName(@Param("name") String name);
List<Dataset> findByStatus(@Param("status") String status);
List<Dataset> findByCreatedBy(@Param("createdBy") String createdBy, RowBounds rowBounds);
Expand Down
5 changes: 5 additions & 0 deletions .../com/datamate/datamanagement/infrastructure/persistence/repository/DatasetRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@
public interface DatasetRepository extends IRepository<Dataset> {
Dataset findByName(String name);

/**
* 使用悲观锁获取数据集（用于更新操作，防止并发冲突）
*/
Dataset getByIdWithLock(String id);

List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList, RowBounds bounds);

long countByCriteria(String type, String status, String keyword, List<String> tagList);
Expand Down
5 changes: 5 additions & 0 deletions ...mate/datamanagement/infrastructure/persistence/repository/impl/DatasetRepositoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ public Dataset findByName(String name) {
return datasetMapper.selectOne(new LambdaQueryWrapper<Dataset>().eq(Dataset::getName, name));
}

@Override
public Dataset getByIdWithLock(String id) {
return datasetMapper.findByIdWithLock(id);
}

@Override
public List<Dataset> findByCriteria(String type, String status, String keyword, List<String> tagList,
RowBounds bounds) {
Expand Down
37 changes: 23 additions & 14 deletions frontend/src/i18n/locales/en/common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@
"updateSuccess": "Task updated successfully",
"updateFailed": "Failed to update task",
"deleteSuccess": "Task deleted",
"deleteConfirm": "Are you sure you want to delete this task? This action cannot be undone.",
"deleteConfirmMessage": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"deleteTitle": "Confirm delete collection task",
"deleteDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"confirmDelete": "Delete",
"cancel": "Cancel"
},
Expand Down Expand Up @@ -375,10 +375,14 @@
"fileDownloadSuccess": "File downloaded successfully"
},
"confirm": {
"deleteDatasetTitle": "Confirm delete this dataset?",
"deleteDatasetTitle": "Confirm delete dataset",
"deleteDatasetDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"deleteFolderTitle": "Confirm delete folder?",
"deleteFolderDesc": "Deleting folder \"{{name}}\" will remove all files and subfolders. This action cannot be undone.",
"deleteFileTitle": "Confirm delete file",
"deleteFileDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"deleteFolderTitle": "Confirm delete folder",
"deleteFolderDesc": "Are you sure you want to delete \"{{itemName}}\" and all its contents? This action cannot be undone.",
"batchDeleteTitle": "Confirm batch delete",
"batchDeleteDesc": "Are you sure you want to delete the selected {{count}} item(s)? This action cannot be undone.",
"deleteConfirm": "Delete",
"deleteCancel": "Cancel"
},
Expand Down Expand Up @@ -664,7 +668,7 @@
"deleteFailed": "Task deletion failed, please try again later"
},
"confirm": {
"deleteTitle": "Confirm delete this task?",
"deleteTitle": "Confirm delete ratio task",
"deleteDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"okText": "Delete",
"cancelText": "Cancel"
Expand Down Expand Up @@ -750,8 +754,8 @@
"delete": "Delete"
},
"confirm": {
"deleteTitle": "Confirm delete this ratio task?",
"deleteDesc": "This task cannot be recovered after deletion. Please proceed with caution.",
"deleteTitle": "Confirm delete ratio task",
"deleteDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone.",
"okText": "Delete",
"cancelText": "Cancel"
},
Expand Down Expand Up @@ -1446,7 +1450,12 @@
"taskDeleted": "Task deleted successfully",
"taskDetailFailed": "Failed to fetch task details",
"taskPaused": "Task paused",
"taskStarted": "Task started"
"taskStarted": "Task started",
"deleteFailed": "Failed to delete task, please try again later"
},
"confirm": {
"deleteTitle": "Confirm delete data processing task",
"deleteDesc": "Are you sure you want to delete \"{{itemName}}\"? This action cannot be undone."
}
},
"template": {
Expand Down Expand Up @@ -2270,8 +2279,8 @@
"cot": "Chain of Thought Generation"
},
"confirm": {
"deleteTitle": "Confirm delete this task?",
"deleteContent": "Task name: {{name}}",
"deleteTitle": "Confirm delete synthesis task",
"deleteDesc": "Are you sure you want to delete \"{{name}}\"? This action cannot be undone.",
"archiveTitle": "Confirm archive this synthesis task?",
"archiveContent": "Task name: {{name}}"
},
Expand Down Expand Up @@ -2567,9 +2576,9 @@
"delete": "Delete Task"
},
"confirm": {
"deleteTitle": "Confirm delete this synthesis task?",
"deleteDescription": "This task cannot be recovered after deletion, please proceed with caution.",
"okText": "Confirm Delete",
"deleteTitle": "Confirm delete synthesis task",
"deleteDescription": "Are you sure you want to delete \"{{name}}\"? This action cannot be undone.",
"okText": "Delete",
"cancelText": "Cancel"
},
"messages": {
Expand Down
41 changes: 25 additions & 16 deletions frontend/src/i18n/locales/zh/common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@
"updateSuccess": "任务更新成功",
"updateFailed": "任务更新失败",
"deleteSuccess": "任务已删除",
"deleteConfirm": "确定要删除该任务吗？此操作不可撤销。",
"deleteConfirmMessage": "确定要删除「{{itemName}}」吗？删除后将无法恢复。",
"deleteTitle": "确认删除归集任务",
"deleteDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？",
"confirmDelete": "删除",
"cancel": "取消"
},
Expand Down Expand Up @@ -375,10 +375,14 @@
"fileDownloadSuccess": "文件下载成功"
},
"confirm": {
"deleteDatasetTitle": "确认删除该数据集？",
"deleteDatasetDesc": "确定要删除「{{itemName}}」吗？删除后将无法恢复。",
"deleteFolderTitle": "确认删除文件夹？",
"deleteFolderDesc": "删除文件夹 \"{{name}}\" 将同时删除其中的所有文件和子文件夹，此操作不可恢复。",
"deleteDatasetTitle": "确认删除数据集",
"deleteDatasetDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？",
"deleteFileTitle": "确认删除文件",
"deleteFileDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？",
"deleteFolderTitle": "确认删除文件夹",
"deleteFolderDesc": "删除「{{itemName}}」及其所有内容后将无法恢复，确定要执行此操作吗？",
"batchDeleteTitle": "确认批量删除",
"batchDeleteDesc": "删除选中的 {{count}} 个项目后将无法恢复，确定要执行此操作吗？",
"deleteConfirm": "删除",
"deleteCancel": "取消"
},
Expand Down Expand Up @@ -664,8 +668,8 @@
"deleteFailed": "任务删除失败，请稍后重试"
},
"confirm": {
"deleteTitle": "确认删除该任务？",
"deleteDesc": "确定要删除「{{itemName}}」吗？删除后将无法恢复。",
"deleteTitle": "确认删除配比任务",
"deleteDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？",
"okText": "删除",
"cancelText": "取消"
},
Expand Down Expand Up @@ -750,8 +754,8 @@
"delete": "删除"
},
"confirm": {
"deleteTitle": "确认删除该配比任务？",
"deleteDesc": "删除后该任务将无法恢复，请谨慎操作。",
"deleteTitle": "确认删除配比任务",
"deleteDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？",
"okText": "删除",
"cancelText": "取消"
},
Expand Down Expand Up @@ -1446,7 +1450,12 @@
"taskDeleted": "任务删除成功",
"taskDetailFailed": "获取任务详情失败",
"taskPaused": "任务已暂停",
"taskStarted": "任务已启动"
"taskStarted": "任务已启动",
"deleteFailed": "任务删除失败，请稍后重试"
},
"confirm": {
"deleteTitle": "确认删除数据处理任务",
"deleteDesc": "删除「{{itemName}}」后将无法恢复，确定要执行此操作吗？"
}
},
"template": {
Expand Down Expand Up @@ -2270,8 +2279,8 @@
"cot": "链式推理生成"
},
"confirm": {
"deleteTitle": "确认删除任务?",
"deleteContent": "任务名：{{name}}",
"deleteTitle": "确认删除合成任务",
"deleteDesc": "删除「{{name}}」后将无法恢复，确定要执行此操作吗？",
"archiveTitle": "确认归档该合成任务?",
"archiveContent": "任务名：{{name}}"
},
Expand Down Expand Up @@ -2567,9 +2576,9 @@
"delete": "删除任务"
},
"confirm": {
"deleteTitle": "确认删除该合成任务？",
"deleteDescription": "删除后将无法恢复，请谨慎操作。",
"okText": "确认删除",
"deleteTitle": "确认删除合成任务",
"deleteDescription": "删除「{{name}}」后将无法恢复，确定要执行此操作吗？",
"okText": "删除",
"cancelText": "取消"
},
"messages": {
Expand Down
Loading
Loading