[缝合]自制COCO格式的关键点检测数据集+训练HRnet_转成coco 关键点数据

一、labelme标注生成Json文件

标注完成后的json文件大概如下：

"shapes": [
        {
            "label": "joint",              # 目标检测框的类别
            "points": [                    # 目标检测框的位置：左上以及右下的坐标（X，Y）
                [
                    767.0000000000005,     # 坐标系 左上角为（0，0）横为x 纵为y
                    900.2692307692307
                ],
                [
                    2028.538461538462,
                    2388.7307692307695
                ]
            ],
            "group_id": null,
            "shape_type": "rectangle",
            "flags": {}
        },
        {
            "label": "patella_1",            # 关键点类别
            "points": [                      # 关键点的坐标
                [    
                    1717.0000000000005,
                    1257.9615384615383
                ]
            ],
            "group_id": null,
            "shape_type": "point",
            "flags": {}
        }
 
]

二、转换labelme的json为coco格式的标注

代码执行的流程：

1、change_name.py

文件的名称都需要改成coco格式的12位数字，空余填充0 ，从0开始，不然最后验证时匹配有问题 

2、divide_jpg_json.py

原图和 json 文件分开到两个文件夹中

3、json2coco_1.py

根据json生成coco格式的标注文件

coco格式解释：

{
    "info"            # 没啥用
    "licenses"        # 没啥用
    "images": [        # 每个图像一个images字段
      {
       "height": 2845,
       "width": 1832,
       "id": 1,
       "file_name": "000000000185.jpg"
      }
     "annotations": [    # 每个框一个annotations字段
      {
       "id": 0,              # 框的id
       "image_id": 1,        # 该框对应的图像的id，需要与image中的id对应
       "category_id": 1,        # 目标检测类别id
       "iscrowd": 0,        # 是否被遮挡，0表示未被遮挡
       "area": 1.0,         # 面积
       "segmentation": null, 
       "bbox": [            # 框的 x,y,width,height
            691.0,
            1089.7,
            986.5,
            1000.2
           ],
       "keypoints": [       # 关键点坐标xyv v=0表示未标注，=1表示不可见 =2表示标注了且可见
            1424.3,
            1285.35,
            2,
            1404.33,
            1602.5,
            2,
            1197.5,
            1812.5,
            2
           ],
       "num_keypoints": 3    # 标注了的关键点数量
      },
    "categories": [
      {
       "supercategory": "joint",
       "id": 1,                    # 目标检测类别号
       "name": "joint",            # 类别名
       "keypoint": [               # 关键点类别
         "patella_1",
         "patella_2",
         "tibia_1"
       ]
      }
}

转换代码如下：

area好像会影响后面计算iou，先标记，还没改正

已修改：

annotation['area'] = self._get_box(bbox)[2]*self._get_box(bbox)[3]  # w*h

自动分割验证集以及训练集:

(我用的是不自动分割)

import os
import sys
import glob
import json
import shutil
import argparse
import numpy as np
import PIL.Image
import os.path as osp
from tqdm import tqdm
from labelme import utils
from sklearn.model_selection import train_test_split
 
 
class Labelme2coco_keypoints():
    def __init__(self, args):
        """
        Lableme 关键点数据集转 COCO 数据集的构造函数:
        Args
            args：命令行输入的参数
                - class_name 根类名字
        """
 
        self.classname_to_id = {args.class_name: 1}
        self.images = []
        self.annotations = []
        self.categories = []
        self.ann_id = 0
        self.img_id = 0
 
    def save_coco_json(self, instance, save_path):
        json.dump(instance, open(save_path, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
 
    def read_jsonfile(self, path):
        with open(path, "r", encoding='utf-8') as f:
            return json.load(f)
 
    def _get_box(self, points):
        min_x = min_y = np.inf
        max_x = max_y = 0
        for x, y in points:
            min_x = min(min_x, x)
            min_y = min(min_y, y)
            max_x = max(max_x, x)
            max_y = max(max_y, y)
        return [min_x, min_y, max_x - min_x, max_y - min_y]
 
    def _get_keypoints(self, points, keypoints, num_keypoints):
        """
        解析 labelme 的原始数据， 生成 coco 标注的 关键点对象
        例如：
            "keypoints": [
                67.06149888292556,  # x 的值
                122.5043507571318,  # y 的值
                1,                  # 相当于 Z 值，如果是2D关键点 0：不可见 1：表示可见。  xxx  1表示标注了但不可见，2表示可见
                82.42582269256718,
                109.95672933232304,
                1,
                ...,
            ],
        """
 
        if points[0] == 0 and points[1] == 0:
            visable = 0
        else:
            visable = 2
            num_keypoints += 1
        keypoints.extend([points[0], points[1], visable])
        return keypoints, num_keypoints
 
    def _image(self, obj, path):
        """
        解析 labelme 的 obj 对象，生成 coco 的 image 对象
        生成包括：id，file_name，height，width 4个属性
        示例：
             {
                "file_name": "training/rgb/00031426.jpg",
                "height": 224,
                "width": 224,
                "id": 31426
            }
        """
 
        image = {}
 
        #img_x = utils.img_b64_to_arr(obj['imageData'])  # 获得原始 labelme 标签的 imageData 属性，并通过 labelme 的工具方法转成 array
        image['height'] = obj['imageHeight']
        image['width'] = obj['imageWidth']  # 获得图片的宽高
 
        # self.img_id = int(os.path.basename(path).split(".json")[0])
        image['id'] = self.img_id
        self.img_id = self.img_id + 1
 
        image['file_name'] = os.path.basename(path).replace(".json", ".jpg")
 
        return image
 
    def _annotation(self, bboxes_list, keypoints_list, json_path):
        """
        生成coco标注
        Args：
            bboxes_list： 矩形标注框
            keypoints_list： 关键点
            json_path：json文件路径
        """
 
        if len(keypoints_list) != args.join_num * len(bboxes_list):
            print('you loss {} keypoint(s) with file {}'.format(args.join_num * len(bboxes_list) - len(keypoints_list), json_path))
            print('Please check ！！！')
            sys.exit()
        i = 0
        for object in bboxes_list:
            annotation = {}
            keypoints = []
            num_keypoints = 0
 
            label = object['label']
            bbox = object['points']
            annotation['id'] = self.ann_id
            annotation['image_id'] = self.img_id-1
            annotation['category_id'] = int(self.classname_to_id[label])
            annotation['iscrowd'] = 0
            annotation['area'] = self._get_box(bbox)[2]*self._get_box(bbox)[3]  # w*h
            annotation['segmentation'] = None #[np.asarray(bbox).flatten().tolist()]
            annotation['bbox'] = self._get_box(bbox)
 
            for keypoint in keypoints_list[i * args.join_num: (i + 1) * args.join_num]:
                point = keypoint['points']
                annotation['keypoints'], num_keypoints = self._get_keypoints(point[0], keypoints, num_keypoints)
            annotation['num_keypoints'] = num_keypoints
 
            i += 1
            self.ann_id += 1
            self.annotations.append(annotation)
 
    def _init_categories(self):
        """
        初始化 COCO 的 标注类别
        例如：
        "categories": [
            {
                "supercategory": "hand",
                "id": 1,
                "name": "hand",
                "keypoints": [
                    "wrist",
                    "thumb1",
                    "thumb2",
                    ...,
                ],
                "skeleton": [
                ]
            }
        ]
        """
 
        for name, id in self.classname_to_id.items():
            category = {}
 
            category['supercategory'] = name
            category['id'] = id
            category['name'] = name
            # 21 个关键点数据
            ['patella_1', 'patella_2', 'tibia_1']
            category['keypoint'] = ['patella_1', 'patella_2', 'tibia_1']
            # category['keypoint'] = [str(i + 1) for i in range(args.join_num)]
 
            self.categories.append(category)
 
    def to_coco(self, json_path_list):
        """
        Labelme 原始标签转换成 coco 数据集格式，生成的包括标签和图像
        Args：
            json_path_list：原始数据集的目录
        """
        [[1082.0, 408.0], [2478.0, 1888.0]]
        self._init_categories()
 
        for json_path in tqdm(json_path_list):
            obj = self.read_jsonfile(json_path)  # 解析一个标注文件
            self.images.append(self._image(obj, json_path))  # 解析图片
            shapes = obj['shapes']  # 读取 labelme shape 标注
 
            bboxes_list, keypoints_list = [], []
            for shape in shapes:
                if shape['shape_type'] == 'rectangle':
                    bboxes_list.append(shape)           # bboxs
                elif shape['shape_type'] == 'point':
                    keypoints_list.append(shape)        # keypoints
 
            self._annotation(bboxes_list, keypoints_list, json_path)
 
        keypoints = {}
        keypoints['images'] = self.images
        keypoints['annotations'] = self.annotations
        keypoints['categories'] = self.categories
        return keypoints
 
def init_dir(base_path):
    """
    初始化COCO数据集的文件夹结构；
    coco - annotations  #标注文件路径
         - train        #训练数据集
         - val          #验证数据集
    Args：
        base_path：数据集放置的根路径
    """
    if not os.path.exists(os.path.join(base_path, "coco", "annotations")):
        os.makedirs(os.path.join(base_path, "coco", "annotations"))
    if not os.path.exists(os.path.join(base_path, "coco", "train")):
        os.makedirs(os.path.join(base_path, "coco", "train"))
    if not os.path.exists(os.path.join(base_path, "coco", "val")):
        os.makedirs(os.path.join(base_path, "coco", "val"))
 
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--class_name", "--n", help="object class name",default='joint',  type=str)
    parser.add_argument("--input", "--i", help="json file path (jsonfiles)",default='/home/zlhaha/WJ/joints_project/yolov8-pose/Project/dataset_raw/COCO/json_file_coco' ,type=str)
    parser.add_argument("--output", "--o", help="output file path (coco format)",default='/home/zlhaha/WJ/joints_project/deep-high-resolution-net.pytorch-master/data', type=str)
    parser.add_argument("--join_num", "--j", help="number of join", default=3 ,type=int)
    parser.add_argument("--ratio", "--r", help="train and test split ratio", type=float, default=0.1)
    args = parser.parse_args()
 
    labelme_path = args.input
    saved_coco_path = args.output
 
    init_dir(saved_coco_path)                   # 初始化COCO数据集的文件夹结构
 
    json_list_path = glob.glob(labelme_path + "/*.json")
    train_path, val_path = train_test_split(json_list_path, test_size=args.ratio)
    print('{} for training'.format(len(train_path)),
          '\n{} for testing'.format(len(val_path)))
    print('Start transform please wait ...')
 
    l2c_train = Labelme2coco_keypoints(args)  # 构造数据集生成类
 
    # 生成训练集
    train_keypoints = l2c_train.to_coco(train_path)
    l2c_train.save_coco_json(train_keypoints, os.path.join(saved_coco_path, "coco", "annotations", "keypoints_train.json"))
 
    # 生成验证集
    l2c_val = Labelme2coco_keypoints(args)
    val_instance = l2c_val.to_coco(val_path)
    l2c_val.save_coco_json(val_instance, os.path.join(saved_coco_path, "coco", "annotations", "keypoints_val.json"))
 
 
    image_root_path = '/home/zlhaha/WJ/joints_project/yolov8-pose/Project/dataset_raw/COCO/dataset_coco'
    # 拷贝 labelme 的原始图片到训练集和验证集里面
    for file in train_path:
        img = os.path.join(image_root_path, file.split('/')[-1].replace("json", "jpg"))
        shutil.copy(img, os.path.join(saved_coco_path, "coco", "train"))
    for file in val_path:
        img = os.path.join(image_root_path, file.split('/')[-1].replace("json", "jpg"))
        shutil.copy(img, os.path.join(saved_coco_path, "coco", "val"))

三、开始改代码训练 HRnet

https://arxiv.org/abs/1904.04514

1、把数据放到data中，以这种格式

2、修改lib/dataset/coco.py 中关节点的类别 （我的是3）

self.num_joints = 3

3、！！！！！cocoeval.py 修改 523行self.kpt_oks_sigmas为自己数据集的，我的是3类。否则汇报以下错误

这里也需要修改成自己的类别数量 sigmas是关节的权重

4、修改np.float为float，原因是numpy版本问题

5、在yaml文件中把FLIP关掉，我用不到就关掉了，不然我的报错

FLIP_TEST FLIP: False 两个变量都设置为false 

6、plot_coco.py 修改

把前面的color等变量都改成自己的数量 我的是3

stick注释掉，我的任务不需要连接线

四：HRnet记录

1、解决路径问题：我要用到pycharm调试，而github代码是用终端跑，所以base路径不同，需要改，对base文件右键，mark directory root 然后把import完善一下

lib.

2、for i, (input, target, target_weight, meta) in enumerate(train_loader):

w32_384x288_adam_lr1e-3  input是288*384，网络的输出是72*96，target也是72*96，target_weight是可见不可见，mete是dict记录一些数据，没啥用

IMAGE_SIZE:
- 288
- 384
HEATMAP_SIZE:
- 72
- 96