Java: MultiDataSource 导出 CVAuto 数据集新增支持按 ratio 参数对应测试集比例切分数据，优化目录结构

Author: TommyLemon

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/java/apijson/DatasetUtil.java

@@ -68,7 +68,7 @@ public static void main(String[] args) {
  System.out.println("\nGenerating dataset from JSONObject data...");
  List<JSONObject> sampleData = createSampleJSONObjectData();
  Set<TaskType> jsonTasks = new HashSet<>(Collections.singletonList(TaskType.DETECTION));
- generate("./output/json_dataset", jsonTasks, sampleData);
+ generate(sampleData, jsonTasks, "./output/detection_dataset/", "train");
 
  } catch (IOException e) {
  e.printStackTrace();
@@ -513,12 +513,13 @@ public static void generate(String outputDir, Set<TaskType> tasks) throws IOExce
 
  /**
  * 从 List<JSONObject> 数据生成 COCO 数据集
- * @param outputDir 输出目录
- * @param tasks 任务类型集合
  * @param data 包含图片和标注信息的 JSONObject 列表
+ * @param tasks 任务类型集合
+ * @param outputDir 输出目录
+ * @param part train/val
  * @throws IOException
  */
- public static void generate(String outputDir, Set<TaskType> tasks, List<JSONObject> data) throws IOException {
+ public static void generate(List<JSONObject> data, Set<TaskType> tasks, String outputDir, String part) throws IOException {
  if (data == null || data.isEmpty()) {
  throw new IllegalArgumentException("Data list cannot be null or empty");
  }
@@ -605,13 +606,15 @@ public static void generate(String outputDir, Set<TaskType> tasks, List<JSONObje
  CocoDataset cocoDataset = builder.build();
 
  // 为不同任务生成不同的文件名
- String taskName = tasks.iterator().next().toString().toLowerCase();
- String outputJsonPath = Paths.get(outputDir, "annotations", "instances_" + taskName + ".json").toString();
+ String[] keys = StringUtil.split(outputDir, "/");
+ String taskName = StringUtil.isNotEmpty(part) ? part : (keys != null && keys.length >= 1 ? keys[keys.length - 1] : tasks.iterator().next().toString().toLowerCase());
 
+ String outputJsonPath = Paths.get(outputDir, "annotations", taskName + ".json").toString();
  writeToFile(cocoDataset, outputJsonPath);
+ writeToFile(cocoDataset, Paths.get(outputDir, taskName + ".json").toString());
 
- // 复制图片文件到指定目录 outputDir/images/
- copyImagesToDirectory(cocoDataset.getImages(), outputDir + "/images/");
+ // 复制图片文件到指定目录 outputDir/images/ train/val
+ copyImagesToDirectory(cocoDataset.getImages(), outputDir + taskName + "/");
 
  System.out.println("Successfully generated dataset from JSONObject data at: " + outputDir);
  }

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/java/apijson/boot/FileController.java

@@ -456,25 +456,29 @@ public ResponseEntity<Object> downloadCVReport(@PathVariable(name = "id") String
 public ResponseEntity<Object> downloadDataset(
 @PathVariable("id") String idStr,
 @RequestParam(name = "type", required = false) String type,
-@RequestParam(name = "datasetName", required = false) String datasetName
+@RequestParam(name = "ratio", required = false) String ratioStr,
+@RequestParam(name = "name", required = false) String datasetName
 ) throws IOException {
 try {
-// 参数验证
+long repOrDocId = Long.parseLong(idStr);
+if (repOrDocId <= 0) {
+throw new IllegalArgumentException("id 必须为 > 0 的 reportId 或 documentId 有效整数！");
+}
+
 if (StringUtil.isNotEmpty(type)) {
 validateCocoType(type);
 } else {
 type = "";
 }
 
-String dataset = StringUtil.isEmpty(datasetName) ? type + "_dataset" : datasetName;
-String exportDir = fileUploadRootDir + dataset + "/";
-
-long repOrDocId = Long.parseLong(idStr);
-if (repOrDocId <= 0) {
-throw new IllegalArgumentException("id 必须为 > 0 的 reportId 或 documentId 有效整数！");
+int ratio = StringUtil.isEmpty(ratioStr) ? 20 : Integer.parseInt(ratioStr);
+if (ratio < 0 || ratio > 100) {
+throw new IllegalArgumentException("测试集比例 ratio 必须为 0 ~ 100 范围内的有效整数！");
 }
 
-String name = "CVAuto_" + dataset + repOrDocId + ".zip";
+String dataset = StringUtil.isNotEmpty(datasetName) ? datasetName : "CVAuto_" + (StringUtil.isNotEmpty(type) ? type + "_" : "") + "dataset_" + repOrDocId;
+String exportDir = fileUploadRootDir + dataset + "/";
+String name = dataset + ".zip";
 String path = fileUploadRootDir + name;
 
 File file = new File(path);
@@ -499,8 +503,8 @@ public ResponseEntity<Object> downloadDataset(
 
 { // [] <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
 JSONObject item = new JSONObject();
-//item.put("count", 0);
-item.put("count", 3);
+item.put("count", 0);
+//item.put("count", 3);
 item.put("join", "@/TestRecord");
 
 { // Random <<<<<<<<<<<<<<<<<<<<<<<<<<<<<
@@ -545,17 +549,29 @@ public ResponseEntity<Object> downloadDataset(
 //long documentId = lastTr == null ? 0 : lastTr.getLongValue("documentId");
 //long randomId = lastTr == null ? 0 : lastTr.getLongValue("randomId");
 if (reportId != repOrDocId) {
-name = "CVAuto_" + dataset + (reportId > 0 ? reportId : repOrDocId + "_last") + ".zip";
+dataset = StringUtil.isNotEmpty(datasetName) ? datasetName : "CVAuto_" + (StringUtil.isNotEmpty(type) ? type + "_" : "") + "dataset_" + (reportId > 0 ? reportId : repOrDocId + "_last");
+exportDir = fileUploadRootDir + dataset + "/";
+name = dataset + ".zip";
 path = fileUploadRootDir + name;
 }
 
 JSONArray array = response.getJSONArray("[]");
 
-List<JSONObject> list = new ArrayList<>();
+List<JSONObject> trainList = new ArrayList<>();
+List<JSONObject> validList = new ArrayList<>();
 if (array != null) {
-for (int i = 0; i < array.size(); i++) {
+int len = array.size();
+for (int i = 0; i < len; i++) {
 JSONObject item = array.getJSONObject(i);
-list.add(item);
+if (item == null || item.isEmpty()) {
+continue;
+}
+
+if (ratio <= 0 || ratio <= 100 - 100.0*i/len) {
+trainList.add(item);
+} else {
+validList.add(item);
+}
 
 //JSONObject random = item == null ? null : item.getJSONObject("Random");
 //JSONObject testRecord = item == null ? null : item.getJSONObject("TestRecord");
@@ -586,7 +602,9 @@ public ResponseEntity<Object> downloadDataset(
 //generateCocoDatasetFromApiJson(exportDir, type, dataset, list);
 
 Set<DatasetUtil.TaskType> detectionTasks = new HashSet<>(Collections.singletonList(DatasetUtil.TaskType.DETECTION));
-DatasetUtil.generate(exportDir, detectionTasks, list);
+
+DatasetUtil.generate(trainList, detectionTasks, exportDir, "train");
+DatasetUtil.generate(validList, detectionTasks, exportDir, "val");
 
 createZipFromDirectory(exportDir, path);
 

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/resources/static/cv/js/main.js

@@ -2949,7 +2949,7 @@ https://github.com/Tencent/APIJSON/issues
  }
  else if (this.isRandomShow && this.isRandomListShow) {
  var id = this.reportId || this.getCurrentDocumentId()
- window.open(this.server + '/download/dataset/' + id)
+ window.open(this.server + '/download/dataset/' + id + "?ratio=20")
  }
  else if (this.view == 'markdown' || this.view == 'output') { //model
  var clazz = StringUtil.trim(this.exTxt.name)