Hello, first time using TAO Toolkit, I’m trying to train a custom model for object detection on aerial imagery from a drone. I took the dataset from here in TFRecord format, and followed the instructions from the tutorial, however when I run the command tao detectnet_v2 train -k custom_model -e ~/specs.txt I get the following error:
Traceback (most recent call last): File "</usr/local/lib/python3.6/dist-packages/iva/detectnet_v2/scripts/train.py>", line 3, in <module> File "<frozen iva.detectnet_v2.scripts.train>", line 1022, in <module> File "<frozen iva.detectnet_v2.scripts.train>", line 1011, in <module> File "<decorator-gen-117>", line 2, in main File "<frozen iva.detectnet_v2.utilities.timer>", line 46, in wrapped_fn File "<frozen iva.detectnet_v2.scripts.train>", line 994, in main File "<frozen iva.detectnet_v2.scripts.train>", line 853, in run_experiment File "<frozen iva.detectnet_v2.scripts.train>", line 625, in train_gridbox File "<frozen iva.detectnet_v2.dataloader.build_dataloader>", line 273, in build_dataloader File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 491, in __init__ File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 548, in _construct_data_sources File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 398, in __init__ File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 439, in _get_max_image_size IndexError: list index (0) out of range Only solution I found was in this thread, but when I tried uploading the dataset to CVAT it stuck at 81%. Does anyone know what else I can try?
This is my spec file:
# Sample model config for to instantiate a resnet18 model with pretrained weights and freeze blocks 0, 1 # with all shortcuts having projection layers. model_config { arch: "mobilenet_v2" pretrained_model_file: '' freeze_blocks: 0 freeze_blocks: 1 all_projections: False num_layers: 18 use_pooling: False use_batch_norm: True dropout_rate: 0.0 objective_set: { cov {} bbox { scale: 35.0 offset: 0.5 } } } ################################################################################################################################################################ # Sample rasterizer configs to instantiate a 3 class bbox rasterizer bbox_rasterizer_config { target_class_config { key: "airport" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "baseball-diamond" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "basketball-court" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "bridge" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "container-crane" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "ground-track-field" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "harbor" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "helicopter" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "helipad" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "large-vehicle" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "plane" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "roundabout" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "ship" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "small-vehicle" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "soccer-ball-field" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "storage-tank" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "swimming-pool" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } target_class_config { key: "tennis-court" value: { cov_center_x: 0.5 cov_center_y: 0.5 cov_radius_x: 0.4 cov_radius_y: 0.4 bbox_min_radius: 1.0 } } deadzone_radius: 0.67 } ################################################################################################################################################################ postprocessing_config { target_class_config { key: "airport" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "baseball-diamond" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "basketball-court" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "bridge" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "container-crane" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "ground-track-field" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "harbor" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "helicopter" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "helipad" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "large-vehicle" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "plane" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "roundabout" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "ship" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "small-vehicle" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "soccer-ball-field" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "storage-tank" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "swimming-pool" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } target_class_config { key: "tennis-court" value: { clustering_config { coverage_threshold: 0.005 dbscan_eps: 0.15 dbscan_min_samples: 0.05 minimum_bounding_box_height: 20 } } } } ################################################################################################################################################################ cost_function_config { target_classes { name: "airport" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "baseball-diamond" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "basketball-court" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "bridge" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "container-crane" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "ground-track-field" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "harbor" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "helicopter" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "helipad" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "large-vehicle" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "plane" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "roundabout" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "ship" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "small-vehicle" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "soccer-ball-field" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "storage-tank" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "swimming-pool" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } target_classes { name: "tennis-court" class_weight: 1.0 coverage_foreground_weight: 0.05 objectives { name: "cov" initial_weight: 1.0 weight_target: 1.0 } objectives { name: "bbox" initial_weight: 10.0 weight_target: 10.0 } } enable_autoweighting: True max_objective_weight: 0.9999 min_objective_weight: 0.0001 } ################################################################################################################################################################ training_config { batch_size_per_gpu: 8 num_epochs: 120 learning_rate { soft_start_annealing_schedule { min_learning_rate: 5e-6 max_learning_rate: 5e-4 soft_start: 0.1 annealing: 0.7 } } regularizer { type: L1 weight: 3e-9 } optimizer { adam { epsilon: 1e-08 beta1: 0.9 beta2: 0.999 } } cost_scaling { enabled: False initial_exponent: 20.0 increment: 0.005 decrement: 1.0 } visualizer { enabled: true num_images: 3 scalar_logging_frequency: 10 infrequent_logging_frequency: 1 target_class_config { key: "airport" value: { coverage_threshold: 0.005 } } target_class_config { key: "baseball-diamond" value: { coverage_threshold: 0.005 } } target_class_config { key: "basketball-court" value: { coverage_threshold: 0.005 } } target_class_config { key: "bridge" value: { coverage_threshold: 0.005 } } target_class_config { key: "container-crane" value: { coverage_threshold: 0.005 } } target_class_config { key: "ground-track-field" value: { coverage_threshold: 0.005 } } target_class_config { key: "harbor" value: { coverage_threshold: 0.005 } } target_class_config { key: "helicopter" value: { coverage_threshold: 0.005 } } target_class_config { key: "helipad" value: { coverage_threshold: 0.005 } } target_class_config { key: "large-vehicle" value: { coverage_threshold: 0.005 } } target_class_config { key: "plane" value: { coverage_threshold: 0.005 } } target_class_config { key: "roundabout" value: { coverage_threshold: 0.005 } } target_class_config { key: "ship" value: { coverage_threshold: 0.005 } } target_class_config { key: "small-vehicle" value: { coverage_threshold: 0.005 } } target_class_config { key: "soccer-ball-field" value: { coverage_threshold: 0.005 } } target_class_config { key: "storage-tank" value: { coverage_threshold: 0.005 } } target_class_config { key: "swimming-pool" value: { coverage_threshold: 0.005 } } target_class_config { key: "tennis-court" value: { coverage_threshold: 0.005 } } } } ################################################################################################################################################################ # Sample augementation config for augmentation_config { preprocessing { output_image_width: 640 output_image_height: 480 output_image_channel: 3 min_bbox_width: 0.0 min_bbox_height: 0.0 } spatial_augmentation { hflip_probability: 0.5 vflip_probability: 0.0 zoom_min: 1.0 zoom_max: 1.0 translate_max_x: 8.0 translate_max_y: 8.0 } color_augmentation { color_shift_stddev: 0.0 hue_rotation_max: 25.0 saturation_shift_max: 0.2 contrast_scale_max: 0.1 contrast_center: 0.5 } } ################################################################################################################################################################ # Sample evaluation config to run evaluation in integrate mode for the given 3 class model, # at every 10th epoch starting from the epoch 1. evaluation_config { average_precision_mode: INTEGRATE validation_period_during_training: 10 first_validation_epoch: 1 minimum_detection_ground_truth_overlap { key: "airport" value: 0.5 } minimum_detection_ground_truth_overlap { key: "baseball-diamond" value: 0.5 } minimum_detection_ground_truth_overlap { key: "basketball-court" value: 0.5 } minimum_detection_ground_truth_overlap { key: "bridge" value: 0.5 } minimum_detection_ground_truth_overlap { key: "container-crane" value: 0.5 } minimum_detection_ground_truth_overlap { key: "ground-trank-field" value: 0.5 } minimum_detection_ground_truth_overlap { key: "harbor" value: 0.5 } minimum_detection_ground_truth_overlap { key: "helicopter" value: 0.5 } minimum_detection_ground_truth_overlap { key: "helipad" value: 0.5 } minimum_detection_ground_truth_overlap { key: "large-vehicle" value: 0.5 } minimum_detection_ground_truth_overlap { key: "plane" value: 0.5 } minimum_detection_ground_truth_overlap { key: "roundabout" value: 0.5 } minimum_detection_ground_truth_overlap { key: "ship" value: 0.5 } minimum_detection_ground_truth_overlap { key: "small-vehicle" value: 0.5 } minimum_detection_ground_truth_overlap { key: "soccer-ball-field" value: 0.5 } minimum_detection_ground_truth_overlap { key: "storage-tank" value: 0.5 } minimum_detection_ground_truth_overlap { key: "swimming-pool" value: 0.5 } minimum_detection_ground_truth_overlap { key: "tennis-court" value: 0.5 } evaluation_box_config { key: "airport" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "baseball-diamond" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "basketball-court" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "bridge" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "container-crane" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "ground-trank-field" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "harbor" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "helicopter" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "helipad" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "large-vehicle" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "plane" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "roundabout" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "ship" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "small-vehicle" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "soccer-ball-field" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "storage-tank" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "swimming-pool" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } evaluation_box_config { key: "tennis-court" value { minimum_height: 4 maximum_height: 9999 minimum_width: 4 maximum_width: 9999 } } } ################################################################################################################################################################ dataset_config { data_sources: { tfrecords_path: "/home/rp-hyperx/Downloads/DOTA v2.0 --easy.v7-small_object.tfrecord/train/no.tfrecord" image_directory_path: "/home/rp-hyperx/Downloads/DOTA v2.0 --easy.v7-small_object.tfrecord/train/images/" } image_extension: "jpg" target_class_mapping { key: "airport" value: "airport" } target_class_mapping { key: "baseball-diamond" value: "baseball-diamond" } target_class_mapping { key: "basketball-court" value: "basketball-court" } target_class_mapping { key: "bridge" value: "bridge" } target_class_mapping { key: "container-crane" value: "container-crane" } target_class_mapping { key: "ground-track-field" value: "ground-track-field" } target_class_mapping { key: "harbor" value: "harbor" } target_class_mapping { key: "helicopter" value: "helicopter" } target_class_mapping { key: "helipad" value: "helipad" } target_class_mapping { key: "large-vehicle" value: "large-vehicle" } target_class_mapping { key: "plane" value: "plane" } target_class_mapping { key: "roundabout" value: "roundabout" } target_class_mapping { key: "ship" value: "ship" } target_class_mapping { key: "small-vehicle" value: "small-vehicle" } target_class_mapping { key: "soccer-ball-field" value: "soccer-ball-field" } target_class_mapping { key: "storage-tank" value: "storage-tank" } target_class_mapping { key: "swimming-pool" value: "swimming-pool" } target_class_mapping { key: "tennis-court" value: "tennis-court" } validation_fold: 0 } Thank you in advance