[AI特训营第三期]基于半监督ppyoloe垃圾车检测

一、项目背景

二、项目任务说明

该项目将采用先进的机器视觉技术和深度学习算法，对垃圾车进行实时监测和识别，便于后续管理。

三、数据说明

本项目数据为卡车倾倒垃圾数据集，为VOC格式，数据集中只给出部分图片的标注信息，并且存在格式错误，数据错误。数据下载地址：下载地址

数据集导航

• truck_waste_dataset/images 存放图片
• truck_waste_dataset/xml_label 存放标注文件

四、代码实现

#解压数据集!unzip data/data198415/archive.zip

#下载PaddleDetection!git clone https://gitee.com/PaddlePaddle/PaddleDetection.git #从gitee上下载速度会快一些

#安装PaddleDetection相关依赖!pip install motmetrics
!pip install pycocotools
!pip install -U scikit-image
!pip install -r PaddleDetection/requirements.txt
!python  PaddleDetection/setup.py install

#导入相关的包import randomimport osimport xml.dom.minidomimport cv2from PIL import Imageimport numpy as npimport pandas as pdimport shutilimport jsonimport globimport matplotlib.pyplot as pltimport matplotlib.patches as patchesimport seaborn as snsfrom matplotlib.font_manager import FontProperties

4.0 清洗数据集

#如果是标准的COCO的数据集，则无需进行清洗转换

#重写xml文件，标签中有空格无法转COCOdef change_one_xml(xml_path, xml_dw, update_content):
    # 打开xml文档
    dom=xml.dom.minidom.parse(xml_path)
    root=dom.documentElement    # 查找修改路劲
    if 1:
        sub1=root.getElementsByTagName('name') 
        # 修改标签内容
        for i in range(len(sub1)):
            sub1[i].firstChild.data  = update_content        # 保存修改
    with open(xml_path,'w') as fh:
        dom.writexml(fh)# 欲修改文件for xml_path in os.listdir('truck_waste_dataset/xml_label'):    if xml_path!='.ipynb_checkpoints':
        xml_dw = './/object/name'
        # 想要修改成什么内容
        update_content = 'Truck_dumping_construction_waste'
        change_one_xml(os.path.join('truck_waste_dataset/xml_label',xml_path), xml_dw, update_content)

#重写数据集图片和xml名称，；路径中有空格无法读取for path in os.listdir('truck_waste_dataset/images'):
    img_path = os.path.join('truck_waste_dataset/images',path)    if ' ' in img_path:
        img_path_ = img_path
        img_path_ = img_path_.replace(' ','_')
        os.rename(img_path,img_path_)for path in os.listdir('truck_waste_dataset/xml_label'):
    img_path = os.path.join('truck_waste_dataset/xml_label',path)    if ' ' in img_path:
        img_path_ = img_path
        img_path_ = img_path_.replace(' ','_')
        os.rename(img_path,img_path_)

#重写xml文件，改正其某些图片宽高信息错误def change_one_xml(xml_path):
    # 打开xml文档
    img_list = os.listdir('truck_waste_dataset/images')
    dom=xml.dom.minidom.parse(xml_path)
    root=dom.documentElement    # 查找修改路劲
    sub1=root.getElementsByTagName('filename')
    sub1[0].firstChild.data  = root.getElementsByTagName('filename')[0].firstChild.data.replace(' ','_')    if   root.getElementsByTagName('filename')[0].firstChild.data.replace(' ','_') in img_list:
        H,W,C = cv2.imread('truck_waste_dataset/images/'+root.getElementsByTagName('filename')[0].firstChild.data.replace(' ','_')).shape
        sub1=root.getElementsByTagName('width')
        sub1[0].firstChild.data  = W
        sub1=root.getElementsByTagName('height')
        sub1[0].firstChild.data  = H        # 保存修改
    with open(xml_path,'w') as fh:
        dom.writexml(fh)# 欲修改文件for xml_path in os.listdir('truck_waste_dataset/xml_label'):    if xml_path!='.ipynb_checkpoints':        # 想要修改成什么内容
        change_one_xml(os.path.join('truck_waste_dataset/xml_label',xml_path))

%cd truck_waste_dataset

# 生成train.txt和val.txtrandom.seed(2020)
xml_dir = 'xml_label'img_dir = 'images'xml_list = os.listdir('xml_label')
path_list = list()
extra_list = list()for img in os.listdir(img_dir):
    img_path = os.path.join(img_dir,img)
    xml_path = os.path.join(xml_dir,img.replace('jpg', 'xml'))    if img.replace('jpg', 'xml') in xml_list:
        path_list.append((img_path, xml_path))    else:
        extra_list.append(img_path)
random.shuffle(path_list)
ratio = 0.8train_f = open('train.txt', 'w') 
val_f = open('val.txt', 'w')
extra_f = open('extra.txt', 'w')for i ,content in enumerate(path_list):
    img, xml = content
    text = img + ' ' + xml + '\n'
    if i < len(path_list) * ratio:
        train_f.write(text)    else:
        val_f.write(text)for i ,content in enumerate(extra_list):
    extra_f.write(content+'\n')
train_f.close()
val_f.close()
extra_f.close()# 根据自己数据类别生成标签文档label = ['Truck_dumping_construction_waste']with open('label_list.txt', 'w') as f:    for text in label:
        f.write(text + '\n')

%cd ~

len(os.listdir('truck_waste_dataset/images')),len(os.listdir('truck_waste_dataset/xml_label'))

#将数据集转化成COCO数据集#生成训练数据!python PaddleDetection/tools/x2coco.py --dataset_type voc \
--dataset_type voc \
--voc_anno_dir truck_waste_dataset \
--voc_anno_list truck_waste_dataset/train.txt \
--voc_label_list truck_waste_dataset/label_list.txt \
--voc_out_name truck_waste_dataset/train.json

#生成验证数据!python PaddleDetection/tools/x2coco.py --dataset_type voc \
--dataset_type voc \
--voc_anno_dir truck_waste_dataset \
--voc_anno_list truck_waste_dataset/val.txt \
--voc_label_list truck_waste_dataset/label_list.txt \
--voc_out_name truck_waste_dataset/val.json

#写入未标注图片#写入extra.jsonimport json
write_json_context=dict()                                                      #写入.json文件的大字典write_json_context['info']= {'description': '', 'url': '', 'version': '', 'year': 2021, 'contributor': '', 'date_created': '2021-07-25'}
write_json_context['categories']=[]
write_json_context['images']=[]
img_pathDir = 'truck_waste_dataset'with open('truck_waste_dataset/extra.txt','r') as fr:
    lines1=fr.readlines()for i,imageFile in enumerate(lines1):
    imagePath = os.path.join(img_pathDir,imageFile)                             #获取图片的绝对路径
    imagePath = imagePath.replace('\n','')
    image = Image.open(imagePath)                                               #读取图片，然后获取图片的宽和高
    W, H = image.size
    img_context={}                                                              #使用一个字典存储该图片信息
    #img_name=os.path.basename(imagePath)
    path = imageFile.split('\n')[0]
    path = path.split('/')[1]                                      #返回path最后的文件名。如果path以/或\结尾，那么就会返回空值
    img_context['file_name']=path
    src_front, src_back = os.path.splitext(imageFile)                           #将文件名和文件格式分开
    img_context['height']=H
    img_context['width']=W
    img_context['id']=i
    write_json_context['images'].append(img_context)
cat_context={}
cat_context['supercategory'] = 'none'cat_context['id'] = 1cat_context['name'] = 'Truck_dumping_construction_waste'write_json_context['categories'].append(cat_context)
name = os.path.join('truck_waste_dataset',"extra"+ '.json')with open(name,'w') as fw:                                                                #将字典信息写入.json文件中
    json.dump(write_json_context,fw,indent=2)

4.1可视化

###解决中文画图问题myfont = FontProperties(fname=r"NotoSansCJKsc-Medium.otf", size=12)
plt.rcParams['figure.figsize'] = (12, 12)
plt.rcParams['font.family']= myfont.get_family()
plt.rcParams['font.sans-serif'] = myfont.get_name()
plt.rcParams['axes.unicode_minus'] = False

# 加载训练集路径TRAIN_DIR = 'truck_waste_dataset/images/'TRAIN_CSV_PATH = 'truck_waste_dataset/train.json'# 加载训练集图片目录train_fns = glob.glob(TRAIN_DIR + '*')print('数据集图片数量: {}'.format(len(train_fns)))

def generate_anno_result(dataset_path, anno_file):
    with open(os.path.join(dataset_path, anno_file)) as f:
        anno = json.load(f)    
    total=[]    for img in anno['images']:
        hw = (img['height'],img['width'])
        total.append(hw)
    unique = set(total)
    
    ids=[]
    images_id=[]    for i in anno['annotations']:
        ids.append(i['id'])
        images_id.append(i['image_id'])    
    # 创建类别标签字典
    category_dic=dict([(i['id'],i['name']) for i in anno['categories']])
    counts_label=dict([(i['name'],0) for i in anno['categories']])    for i in anno['annotations']:
        counts_label[category_dic[i['category_id']]] += 1
    label_list = counts_label.keys()    # 各部分标签
    size = counts_label.values()    # 各部分大小

    train_fig = pd.DataFrame(anno['images'])
    train_anno = pd.DataFrame(anno['annotations'])
    df_train = pd.merge(left=train_fig, right=train_anno, how='inner', left_on='id', right_on='image_id')
    df_train['bbox_xmin'] = df_train['bbox'].apply(lambda x: x[0])
    df_train['bbox_ymin'] = df_train['bbox'].apply(lambda x: x[1])
    df_train['bbox_w'] = df_train['bbox'].apply(lambda x: x[2])
    df_train['bbox_h'] = df_train['bbox'].apply(lambda x: x[3])
    df_train['bbox_xcenter'] = df_train['bbox'].apply(lambda x: (x[0]+0.5*x[2]))
    df_train['bbox_ycenter'] = df_train['bbox'].apply(lambda x: (x[1]+0.5*x[3]))    print('最小目标面积（像素）：', min(df_train.area))

    balanced = ''
    small_object = ''
    densely = ''
    # 判断样本是否均衡，给出结论
    if max(size) > 5 * min(size):        print('样本不均衡')
        balanced = 'c11'
    else:        print('样本均衡')
        balanced = 'c10'
    # 判断样本是否存在小目标，给出结论
    if min(df_train.area) < 900:        print('存在小目标')
        small_object = 'c21'
    else:        print('不存在小目标')
        small_object = 'c20'
    arr1=[]
    arr2=[]
    x=[]
    y=[]
    w=[]
    h=[]    for index, row in df_train.iterrows():        if index < 1000:            # 获取并记录坐标点
            x.append(row['bbox_xcenter'])
            y.append(row['bbox_ycenter'])
            w.append(row['bbox_w'])
            h.append(row['bbox_h'])    for i in range(len(x)):
        l = np.sqrt(w[i]**2+h[i]**2)
        arr2.append(l)        for j in range(len(x)):
            a=np.sqrt((x[i]-x[j])**2+(y[i]-y[j])**2)            if a != 0:
                arr1.append(a)
    arr1=np.matrix(arr1)    # print(arr1.min())
    # print(np.mean(arr2))
    # 判断是否密集型目标，具体逻辑还需优化
    if arr1.min() <  np.mean(arr2):        print('密集型目标')
        densely = 'c31'
    else:        print('非密集型目标')
        densely = 'c30'
    return balanced, small_object, densely

# 分析训练集数据generate_anno_result('truck_waste_dataset', 'train.json')

最小目标面积（像素）： 1054.0
样本均衡
不存在小目标
密集型目标

('c10', 'c20', 'c31')

# 读取训练集标注文件with open('truck_waste_dataset/train.json', 'r', encoding='utf-8') as f:
    train_data = json.load(f)
train_fig = pd.DataFrame(train_data['images'])

ps = np.zeros(len(train_fig))for i in range(len(train_fig)):
    ps[i]=train_fig['width'][i] * train_fig['height'][i]/1e6plt.title('训练集图片大小分布', fontproperties=myfont)
sns.distplot(ps, bins=21,kde=False)

!python box_distribution.py --json_path truck_waste_dataset/train.json

# 训练集目标大小统计结果train_anno = pd.DataFrame(train_data['annotations'])
df_train = pd.merge(left=train_fig, right=train_anno, how='inner', left_on='id', right_on='image_id')
df_train['bbox_xmin'] = df_train['bbox'].apply(lambda x: x[0])
df_train['bbox_ymin'] = df_train['bbox'].apply(lambda x: x[1])
df_train['bbox_w'] = df_train['bbox'].apply(lambda x: x[2])
df_train['bbox_h'] = df_train['bbox'].apply(lambda x: x[3])
df_train['bbox_xcenter'] = df_train['bbox'].apply(lambda x: (x[0]+0.5*x[2]))
df_train['bbox_ycenter'] = df_train['bbox'].apply(lambda x: (x[1]+0.5*x[3]))
df_train.area.describe()

df_train['bbox_count'] = df_train.apply(lambda row: 1 if any(row.bbox) else 0, axis=1)
train_images_count = df_train.groupby('file_name').sum().reset_index()
plt.title('训练集目标个数分布', fontproperties=myfont)
sns.distplot(train_images_count['bbox_count'], bins=21,kde=True)

4.2模型介绍

PaddleDetection团队结合 Dense Teacher前沿算法，针对 PP-YOLOE+提供了半监督学习方案。

半监督学习结合有标签数据和无标签数据，在大幅节省数据标注的情况下，依然达到较高的模型精度。在实际产业应用过程中，半监督学习是项目冷启动时常见的策略之一。

TalkMe

与AI语伴聊天，练习外语口语

下载

下表是，在仅采用5%、10%有标签数据进行监督学习，95%、90%无标签数据进行半监督学习的情况下，精度得到了1.2~2.5的提升。

配置文件

如果训练小样本算法，修改configs/coco_detection.yml配置文件中的路径

metric: COCOnum_classes: 2TrainDataset:
  name: COCODataSet
  image_dir: images
  anno_path: train.json
  dataset_dir: truck_waste_dataset
  data_fields: ['image', 'gt_bbox', 'gt_class', 'is_crowd']EvalDataset:
  name: COCODataSet
  image_dir: images
  anno_path: val.json
  dataset_dir: truck_waste_dataset
  allow_empty: trueTestDataset:
  name: ImageFolder
  anno_path: annotations/instances_val2017.json # also support txt (like VOC's label_list.txt)
  dataset_dir: dataset/coco # if set, anno_path will be 'dataset_dir/anno_path、

如果是半监督算法，则需要修改configs/semi_det/base/coco_detection_percent_10.yml配置文件中路径

metric: COCOnum_classes: 2# partial labeled COCO, use `SemiCOCODataSet` rather than `COCODataSet`TrainDataset:
  !SemiCOCODataSet
    image_dir: images
    anno_path: train.json
    dataset_dir: truck_waste_dataset
    data_fields: ['image', 'gt_bbox', 'gt_class', 'is_crowd']# partial unlabeled COCO, use `SemiCOCODataSet` rather than `COCODataSet`UnsupTrainDataset:
  !SemiCOCODataSet
    image_dir: images
    anno_path: extra.json
    dataset_dir: truck_waste_dataset
    data_fields: ['image']    supervised: FalseEvalDataset:
  !COCODataSet
    image_dir: images
    anno_path: val.json
    dataset_dir: truck_waste_dataset
    allow_empty: trueTestDataset:
  !ImageFolder
    anno_path: pre.json # also support txt (like VOC's label_list.txt)
    dataset_dir:  # if set, anno_path will be 'dataset_dir/anno_path'

4.3模型训练

#小样本方案# !python PaddleDetection/tools/train.py -c configs/ppyoloe_plus_crn_s_80e_contrast_pcb.yml --use_vdl=True --eval

# 开始训练,训练环境为单卡V100(32G)#半监督方案!python PaddleDetection/tools/train.py -c configs/semi_det/denseteacher/denseteacher_ppyoloe_plus_crn_l_coco_semi010.yml --use_vdl=True --eval --amp

训练结果可视化

4.4模型评价

!python PaddleDetection/tools/eval.py -c configs/semi_det/denseteacher/denseteacher_ppyoloe_plus_crn_l_coco_semi010.yml

4.5模型推理

!python  PaddleDetection/tools/infer.py -c configs/semi_det/denseteacher/denseteacher_ppyoloe_plus_crn_l_coco_semi010.yml \
--infer_img truck_waste_dataset/images/ction-garbage-process-concrete-recycling-crushing-and-recycling-of-c-2ATK81G.jpg\
--draw_threshold 0.1\

4.6模型导出

!python PaddleDetection/tools/export_model.py -c configs/semi_det/denseteacher/denseteacher_ppyoloe_plus_crn_l_coco_semi010.yml

五、效果展示

方案效果对比