[数据集][PASCAL VOC]07+12

综合PASCAL VOC 20072012数据集,进行分类/检测任务。分两步完成:

  1. 下载07 trainval、07 test、12 trainval数据集,解析出分类/检测需要的数据
  2. 根据具体任务(分类或者检测)从中提取数据

相关内容参考:

简介

完成07 + 12数据集合并后,共得到如下数据:

  1. 训练数据:16551张图像,共40058个目标
  2. 测试素据:4952张图像,共12032个目标

数据集下载

下载完成后解压到同一文件夹下,其路径如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
├── VOCtest_06-Nov-2007
│   └── VOCdevkit
│   └── VOC2007
│   ├── Annotations
│   ├── ImageSets
│   ├── JPEGImages
│   ├── SegmentationClass
│   └── SegmentationObject
├── VOCtest_06-Nov-2007.tar
├── VOCtrainval_06-Nov-2007
│   └── VOCdevkit
│   └── VOC2007
│   ├── Annotations
│   ├── ImageSets
│   ├── JPEGImages
│   ├── SegmentationClass
│   └── SegmentationObject
├── VOCtrainval_06-Nov-2007.tar
├── VOCtrainval_11-May-2012
│   └── VOCdevkit
│   └── VOC2012
│   ├── Annotations
│   ├── ImageSets
│   ├── JPEGImages
│   ├── SegmentationClass
│   └── SegmentationObject
└── VOCtrainval_11-May-2012.tar

数据集创建

不同数据集下的标注文件、图像以及txt文件(该文件保存了指定的图像名)位于

  • 2007 test
    • 标注文件:VOCtest_06-Nov-2007/VOCdevkit/VOC2007/Annotations
    • 图像:VOCtest_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages
    • txt文件:VOCtest_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/test.txt
  • 2007 trainval
    • 标注文件:VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/Annotations
    • 图像:VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages
    • txt文件: VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt
  • 2012 trainval
    • 标注文件:VOCtrainval_11-May-2012/VOCdevkit/VOC2012/Annotations
    • 图像:VOCtrainval_11-May-2012/VOCdevkit/VOC2012/JPEGImages
    • txt文件:VOCtrainval_11-May-2012/VOCdevkit/VOC2012/ImageSets/Main/trainval.txt

创建新文件夹pascal-voc,训练数据保存为2007 trainval + 2012 trainval,测试数据保存为2007 test

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# -*- coding: utf-8 -*-

"""
@date: 2020/4/6 下午4:14
@file: pascal_voc.py
@author: zj
@description: 解析07+12数据集
"""

import os
import cv2
import shutil
import numpy as np
from torchvision.datasets import VOCDetection

# 2007 trainval

trainval_07_annotations = '../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/Annotations'
trainval_07_image = '../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages'
trainval_07_txt = '../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt'

# 2007 test

test_07_annotations = '../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/Annotations'
test_07_image = '../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages'
test_07_txt = '../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/test.txt'

# 2012 trainval

trainval_12_annotations = '../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/Annotations'
trainval_12_image = '../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/JPEGImages'
trainval_12_txt = '../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/ImageSets/Main/trainval.txt'


def check_dir(data_dir):
if not os.path.exists(data_dir):
os.mkdir(data_dir)


def parse_data(txt_path, annotation_dir, image_dir):
"""
解析txt文件,返回相应的图像和标注文件
:return:
"""
name_list = np.loadtxt(txt_path, dtype=np.str, delimiter=' ')
print(name_list)

annotation_list = [os.path.join(annotation_dir, name + ".xml") for name in name_list]
image_list = [os.path.join(image_dir, name + ".jpg") for name in name_list]

return name_list, annotation_list, image_list


if __name__ == '__main__':
data_dir = '../../data/pascal-voc'
check_dir(data_dir)

txt_list = [trainval_07_txt, trainval_12_txt, test_07_txt]
annotation_list = [trainval_07_annotations, trainval_12_annotations, test_07_annotations]
image_list = [trainval_07_image, trainval_12_image, test_07_image]

total_train_list = list()
total_test_list = list()

for txt_path, annotation_dir, image_dir in zip(txt_list, annotation_list, image_list):
print(txt_path, annotation_dir, image_dir)
name_list, annotation_list, image_list = parse_data(txt_path, annotation_dir, image_dir)

if 'trainval' in txt_path:
suffix = 'train'
total_train_list.extend(name_list)
else:
suffix = 'test'
total_test_list.extend(name_list)

# 新建结果文件夹
dst_dir = os.path.join(data_dir, suffix)
check_dir(dst_dir)
dst_annotation_dir = os.path.join(dst_dir, 'Annotations')
check_dir(dst_annotation_dir)
dst_image_dir = os.path.join(dst_dir, 'JPEGImages')
check_dir(dst_image_dir)

# 依次复制标注文件和图像
for name, src_annotation_path, src_image_path in zip(name_list, annotation_list, image_list):
dst_annotation_path = os.path.join(dst_annotation_dir, name + ".xml")
dst_image_path = os.path.join(dst_image_dir, name + ".jpg")

shutil.copyfile(src_annotation_path, dst_annotation_path)
shutil.copyfile(src_image_path, dst_image_path)

print('train num: {}, test num: {}'.format(len(total_train_list), len(total_test_list)))

# 保存文件名
train_dir = os.path.join(data_dir, 'train', 'name.csv')
np.savetxt(train_dir, total_train_list, fmt='%s', delimiter=' ')
test_dir = os.path.join(data_dir, 'test', 'name.csv')
np.savetxt(test_dir, total_test_list, fmt='%s', delimiter=' ')

print('done')
########################### output
../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt ../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/Annotations ../../data/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages
['000005' '000007' '000009' ... '009958' '009959' '009961']
../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/ImageSets/Main/trainval.txt ../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/Annotations ../../data/VOCtrainval_11-May-2012/VOCdevkit/VOC2012/JPEGImages
['2008_000002' '2008_000003' '2008_000007' ... '2011_003274' '2011_003275'
'2011_003276']
../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/test.txt ../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/Annotations ../../data/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages
['000001' '000002' '000003' ... '009960' '009962' '009963']
train num: 16551, test num: 4952
done

共得到16551张训练图像和4952张测试图像

分类数据集

从合并后的数据集中解析出分类任务的训练和测试数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
# -*- coding: utf-8 -*-

"""
@date: 2020/3/26 下午2:50
@file: create_train_val.py
@author: zj
@description: 提取分类任务的训练/测试集,分类别保存
"""

import cv2
import numpy as np
import os
import xmltodict

alphabets = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable',
'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']


def check_dir(data_dir):
if not os.path.exists(data_dir):
os.mkdir(data_dir)


def find_all_cate_rects(annotation_dir, name_list):
"""
找出所有的类别的标注框(取消标记为difficult的边界框)
"""
cate_list = list()
for i in range(20):
cate_list.append(list())

for name in name_list:
annotation_path = os.path.join(annotation_dir, name + ".xml")
with open(annotation_path, 'rb') as f:
xml_dict = xmltodict.parse(f)
# print(xml_dict)

objects = xml_dict['annotation']['object']
if isinstance(objects, list):
for obj in objects:
obj_name = obj['name']
obj_idx = alphabets.index(obj_name)

difficult = int(obj['difficult'])
if difficult != 1:
bndbox = obj['bndbox']
cate_list[obj_idx].append({'img_name': name, 'rect':
(int(bndbox['xmin']), int(bndbox['ymin']), int(bndbox['xmax']), int(bndbox['ymax']))})
elif isinstance(objects, dict):
obj_name = objects['name']
obj_idx = alphabets.index(obj_name)

difficult = int(objects['difficult'])
if difficult != 1:
bndbox = objects['bndbox']
cate_list[obj_idx].append({'img_name': name, 'rect':
(int(bndbox['xmin']), int(bndbox['ymin']), int(bndbox['xmax']), int(bndbox['ymax']))})
else:
pass

return cate_list


def save_cate(cate_list, image_dir, res_dir):
"""
保存裁剪的图像
"""
# 保存image_dir下所有图像,以便后续查询
# 前提条件:足够的内存!!!
# image_dict = dict()
# image_name_list = os.listdir(image_dir)
# for name in image_name_list:
# image_path = os.path.join(image_dir, name)
# img = cv2.imread(image_path)
#
# image_dict[name.split('.')[0]] = img

# 遍历所有类别,保存标注的图像
for i in range(20):
cate_name = alphabets[i]
cate_dir = os.path.join(res_dir, cate_name)
check_dir(cate_dir)

for item in cate_list[i]:
img_name = item['img_name']
xmin, ymin, xmax, ymax = item['rect']

image_path = os.path.join(image_dir, img_name+'.jpg')
img = cv2.imread(image_path)
rect_img = img[ymin:ymax, xmin:xmax]
# rect_img = image_dict[img_name][ymin:ymax, xmin:xmax]
img_path = os.path.join(cate_dir, '%s-%d-%d-%d-%d.png' % (img_name, xmin, ymin, xmax, ymax))
cv2.imwrite(img_path, rect_img)


if __name__ == '__main__':
root_dir = '../../data/pascal-voc/'
train_txt_path = '../../data/pascal-voc/train/name.csv'
val_txt_path = '../../data/pascal-voc/test/name.csv'

for phase in ['train', 'test']:
if phase == 'train':
suffix = 'train_imgs'
else:
suffix = 'test_imgs'
dst_dir = os.path.join(root_dir, suffix)
check_dir(dst_dir)
print(dst_dir)

name_path = os.path.join(root_dir, phase, 'name.csv')
name_list = np.loadtxt(name_path, dtype=np.str, delimiter=' ')

annotation_dir = os.path.join(root_dir, phase, 'Annotations')
rects_list = find_all_cate_rects(annotation_dir, name_list)

total_num = 0
# 打印出每个类别的数据
for i in range(20):
total_num += len(rects_list[i])
print(alphabets[i], len(rects_list[i]))
print('total {} num: {}'.format(phase, total_num))

image_dir = os.path.join(root_dir, phase, 'JPEGImages')
save_cate(rects_list, image_dir, dst_dir)

print()
print('done')
########################### 输出
../../data/pascal-voc/train_imgs
aeroplane 1171
bicycle 1064
bird 1605
boat 1140
bottle 1764
bus 822
car 3267
cat 1593
chair 3152
cow 847
diningtable 824
dog 2025
horse 1072
motorbike 1052
person 13256
pottedplant 1487
sheep 1070
sofa 814
train 925
tvmonitor 1108
total train num: 40058
../../data/pascal-voc/test_imgs
aeroplane 285
bicycle 337
bird 459
boat 263
bottle 469
bus 213
car 1201
cat 358
chair 756
cow 244
diningtable 206
dog 489
horse 348
motorbike 325
person 4528
pottedplant 480
sheep 242
sofa 239
train 282
tvmonitor 308
total test num: 12032
done

得到40058张训练图像以及12032张测试图像

坚持原创技术分享,您的支持将鼓励我继续创作!