頻繁設置CGroup觸發linux內核bug導致CGroup running task不調度

#include <iostream>
#include <sys/types.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <pthread.h>
#include <sys/time.h>
#include <string>

using namespace std;
std::string sub_cgroup_dir("/sys/fs/cgroup/cpu/test");

// common lib
bool is_dir(const std::string& path)
{
    struct stat statbuf;
    if (stat(path.c_str(), &statbuf) == 0 )
    {
        if (0 != S_ISDIR(statbuf.st_mode))
        {
            return true;
        }
    }
    return false;
}

bool write_file(const std::string& file_path, int num)
{
    FILE* fp = fopen(file_path.c_str(), "w");
    if (fp = NULL)
    {
        return false;
    }

    std::string write_data = to_string(num);
    fputs(write_data.c_str(), fp);
    fclose(fp);
    return true;
}

// ms
long get_ms_timestamp()
{
    timeval tv;
    gettimeofday(&tv, NULL);
    return (tv.tv_sec * 1000 + tv.tv_usec / 1000);
}

// cgroup
bool create_cgroup()
{
    if (is_dir(sub_cgroup_dir) == false)
    {
        if (mkdir(sub_cgroup_dir.c_str(), S_IRWXU | S_IRGRP) != 0)
        {
            cout << "mkdir cgroup dir fail" << endl;
            return false;
        }
    }

    int pid = getpid();
    cout << "pid is " << pid << endl;
    std::string procs_path = sub_cgroup_dir + "/cgroup.procs";
    return write_file(procs_path, pid);
}

bool set_period(int period)
{
    std::string period_path = sub_cgroup_dir + "/cpu.cfs_period_us";
    return write_file(period_path, period);
}

bool set_quota(int quota)
{
    std::string quota_path = sub_cgroup_dir + "/cpu.cfs_quota_us";
    return write_file(quota_path, quota);
}

// thread
// param: ms interval
void* thread_func(void* param)
{
    int i = 0;
    int interval = (long)param;
    long last = get_ms_timestamp();

    while (true)
    {
        i++;
        if (i % 1000 != 0)
        {
            continue;
        }

        long current = get_ms_timestamp();
        if ((current - last) >= interval)
        {
            usleep(1000);
            last = current;
        }
    }

    pthread_exit(NULL);
}

 void test_thread()
 {
    const int k_thread_num = 10;
    pthread_t pthreads[k_thread_num];

    for (int i = 0; i < k_thread_num; i++)
    {
        if (pthread_create(&pthreads[i], NULL, thread_func, (void*)(i + 1)) != 0)
        {
            cout << "create thread fail" << endl;
        }
        else
        {
            cout << "create thread success,tid is " << pthreads[i] << endl;
        }
    }
}

//argv[0] : period
//argv[1] : quota
int main(int argc,char* argv[])
{
    if (argc <3)
    {
        cout << "usage : ./inactive timer $period $quota" << endl;
        return -1;
    }

    int period = stoi(argv[1]);
    int quota = stoi(argv[2]);
    cout << "period is " << period << endl;
    cout << "quota is " << quota << endl;

    test_thread();
    if (create_cgroup() == false)
    {
        cout << "create cgroup fail" << endl;
        return -1;
    }

    int i =0;
    while (true)
    {
        if (i > 20)
        {
            i = 0;
        }

        i++;
        long current = get_ms_timestamp();
        long last = current;
        while ((current - last) < i)
        {
            usleep(1000);
            current = get_ms_timestamp();
        }
        
        set_period(period);
        set_quota(quota);
    }

    return 0;
}

2.1.2 編譯

g++ -std=c++11 -lpthread trigger_cgroup_timer_inactive.cpp -o inactive_timer

2.1.3 在CentOS7.0~7.5的系統上執行程式

./inactive_timer 100000 10000

tg_set_cfs_quota()
    tg_set_cfs_bandwidth()
            /* restart the period timer (if active) to handle new period expiry */
            if (runtime_enabled && cfs_b->timer_active) {
                /* force a reprogram */
                cfs_b->timer_active = 0;
                __start_cfs_bandwidth(cfs_b);
            }