Inference Error with custom yolov8 model

[junho2000]

Hi i encountered error when i inference with my custom model
as you can see there are a lot of bounding box and wrong inference.(all inference is round-about class)

i followed Normal.md and export ONNX with ultralytics and export TensorRT engine with trtexec
so inference with yolov8s.pt works so good.

but when i execute with my model it works bad
i changed class names, color and num_labels
https://drive.google.com/file/d/13fV2zgIZAsq6ubgGY1CbQld6RmQIe3s_/view?usp=sharing this is my model

//
// Created by ubuntu on 1/20/23.
//
#include "chrono"
#include "opencv2/opencv.hpp"
#include "yolov8.hpp"

const std::vector<std::string> CLASS_NAMES = {
    "crosswork", "highway-entrance", "highway-exit", "no-entry-road", "obstacle-vehicle", "one-way-road", "parking-sign",
    "pedestrian","priority", "round-about", "stop", "traffic-light"};

const std::vector<std::vector<unsigned int>> COLORS = {
    {0, 0, 128},     {0, 0, 170},     {0, 0, 212},     {0, 0, 255},     {0, 0, 0},       {36, 36, 36},
    {73, 73, 73},    {109, 109, 109}, {146, 146, 146}, {182, 182, 182}, {219, 219, 219}, {0, 114, 189}};

int main(int argc, char** argv)
{
    // cuda:0
    cudaSetDevice(0);

    const std::string engine_file_path{argv[1]};
    const std::string path{argv[2]};

    std::vector<std::string> imagePathList;
    bool                     isVideo{false};

    assert(argc == 3);

    auto yolov8 = new YOLOv8(engine_file_path);
    yolov8->make_pipe(true);

    if (IsFile(path)) {
        std::string suffix = path.substr(path.find_last_of('.') + 1);
        if (suffix == "jpg" || suffix == "jpeg" || suffix == "png") {
            imagePathList.push_back(path);
        }
        else if (suffix == "mp4" || suffix == "avi" || suffix == "m4v" || suffix == "mpeg" || suffix == "mov"
                 || suffix == "mkv") {
            isVideo = true;
        }
        else {
            printf("suffix %s is wrong !!!\n", suffix.c_str());
            std::abort();
        }
    }
    else if (IsFolder(path)) {
        cv::glob(path + "/*.jpg", imagePathList);
    }

    cv::Mat  res, image;
    cv::Size size        = cv::Size{640, 640};
    int      num_labels  = 12;
    int      topk        = 100;
    float    score_thres = 0.25f;
    float    iou_thres   = 0.65f;

    std::vector<Object> objs;

    cv::namedWindow("result", cv::WINDOW_AUTOSIZE);

    std::cout << "loop start" << std::endl;
    if (isVideo) {
        cv::VideoCapture cap(path);

        if (!cap.isOpened()) {
            printf("can not open %s\n", path.c_str());
            return -1;
        }
        while (cap.read(image)) {
            objs.clear();
            yolov8->copy_from_Mat(image, size);
            auto start = std::chrono::system_clock::now();
            yolov8->infer();
            auto end = std::chrono::system_clock::now();
            yolov8->postprocess(objs, score_thres, iou_thres, topk, num_labels);
            yolov8->draw_objects(image, res, objs, CLASS_NAMES, COLORS);
            auto tc = (double)std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000.;
            printf("cost %2.4lf ms\n", tc);
            cv::imshow("result", res);
            if (cv::waitKey(10) == 'q') {
                break;
            }
        }
    }
    else {
        for (auto& path : imagePathList) {
            objs.clear();
            image = cv::imread(path);
            yolov8->copy_from_Mat(image, size);
            auto start = std::chrono::system_clock::now();
	        std::cout << "infer started" << std::endl;
            yolov8->infer();
	        std::cout << "infer ended" << std::endl;
            auto end = std::chrono::system_clock::now();
            yolov8->postprocess(objs, score_thres, iou_thres, topk, num_labels);
            yolov8->draw_objects(image, res, objs, CLASS_NAMES, COLORS);
            auto tc = (double)std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000.;
            printf("cost %2.4lf ms\n", tc);
            cv::imshow("result", res);
            cv::waitKey(0);
        }
    }
    cv::destroyAllWindows();
    delete yolov8;
    return 0;
}

thank you for considering my question

[triple-Mu]

Hello could you please upload your onnx or pt?
Also upload a test image please.

[junho2000]

https://drive.google.com/file/d/13fV2zgIZAsq6ubgGY1CbQld6RmQIe3s_/view?usp=sharing this is my model(best.pt)

[triple-Mu]

https://drive.google.com/file/d/13fV2zgIZAsq6ubgGY1CbQld6RmQIe3s_/view?usp=sharing this is my model(best.pt)

Sorry for replying to you so late.
I test it works well now.
How about you now?