00001 //--------------------------------------------------------------------------
00002 // Copyright (C) 2016-2017 Cisco and/or its affiliates. All rights reserved.
00003 //
00004 // This program is free software; you can redistribute it and/or modify it
00005 // under the terms of the GNU General Public License Version 2 as published
00006 // by the Free Software Foundation. You may not use, modify or distribute
00007 // this program under any other version of the GNU General Public License.
00008 //
00009 // This program is distributed in the hope that it will be useful, but
00010 // WITHOUT ANY WARRANTY; without even the implied warranty of
00011 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
00012 // General Public License for more details.
00013 //
00014 // You should have received a copy of the GNU General Public License along
00015 // with this program; if not, write to the Free Software Foundation, Inc.,
00016 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
00017 //--------------------------------------------------------------------------
00018 // thread_config.cc author Michael Altizer <mialtize@cisco.com>
00019
00020 #ifdef HAVE_CONFIG_H
00021 #include "config.h"
00022 #endif
00023
00024 #include "thread_config.h"
00025
00026 #include <hwloc.h>
00027
00028 #include "log/messages.h"
00029 #include "utils/util.h"
00030
00031 #ifdef UNIT_TEST
00032 #include "catch/catch.hpp"
00033 #endif
00034
00035 static hwloc_topology_t topology = nullptr;
00036 static hwloc_cpuset_t process_cpuset = nullptr;
00037 static const struct hwloc_topology_support* topology_support = nullptr;
00038 static unsigned instance_max = 1;
00039
00040 struct CpuSet
00041 {
00042 CpuSet(hwloc_cpuset_t set) : cpuset(set) { }
00043 ~CpuSet()
00044 {
00045 if (cpuset)
00046 hwloc_bitmap_free(cpuset);
00047 }
00048
00049 hwloc_cpuset_t cpuset;
00050 };
00051
00052 bool ThreadConfig::init()
00053 {
00054 if (hwloc_topology_init(&topology))
00055 return false;
00056 if (hwloc_topology_load(topology))
00057 {
00058 hwloc_topology_destroy(topology);
00059 return false;
00060 }
00061 topology_support = hwloc_topology_get_support(topology);
00062 if (topology_support->cpubind->get_thisproc_cpubind)
00063 {
00064 process_cpuset = hwloc_bitmap_alloc();
00065 hwloc_get_cpubind(topology, process_cpuset, HWLOC_CPUBIND_PROCESS);
00066 }
00067 else
00068 process_cpuset = hwloc_bitmap_dup(hwloc_topology_get_allowed_cpuset(topology));
00069 return true;
00070 }
00071
00072 void ThreadConfig::set_instance_max(unsigned max)
00073 {
00074 if (max)
00075 instance_max = max;
00076 else
00077 {
00078 /* A max of 0 indicates automatic allocation. Set the instance max to the total number of
00079 CPUs in the our process's running cpuset. */
00080 instance_max = hwloc_bitmap_weight(process_cpuset);
00081 }
00082 }
00083
00084 unsigned ThreadConfig::get_instance_max()
00085 {
00086 return instance_max;
00087 }
00088
00089 CpuSet* ThreadConfig::validate_cpuset_string(const char* cpuset_str)
00090 {
00091 hwloc_bitmap_t cpuset = hwloc_bitmap_alloc();
00092 if (hwloc_bitmap_list_sscanf(cpuset, cpuset_str) ||
00093 !hwloc_bitmap_isincluded(cpuset, process_cpuset))
00094 {
00095 hwloc_bitmap_free(cpuset);
00096 return nullptr;
00097 }
00098 return new CpuSet(cpuset);
00099 }
00100
00101 void ThreadConfig::destroy_cpuset(CpuSet *cpuset)
00102 {
00103 delete cpuset;
00104 }
00105
00106 void ThreadConfig::term()
00107 {
00108 if (topology)
00109 {
00110 hwloc_topology_destroy(topology);
00111 topology = nullptr;
00112 }
00113 if (process_cpuset)
00114 {
00115 hwloc_bitmap_free(process_cpuset);
00116 process_cpuset = nullptr;
00117 }
00118 topology_support = nullptr;
00119 }
00120
00121 ThreadConfig::~ThreadConfig()
00122 {
00123 for (auto iter = thread_affinity.begin(); iter != thread_affinity.end(); iter++)
00124 delete iter->second;
00125 }
00126
00127 void ThreadConfig::set_thread_affinity(SThreadType type, unsigned id, CpuSet* cpuset)
00128 {
00129 if (topology_support->cpubind->set_thisthread_cpubind)
00130 {
00131 TypeIdPair key { type, id };
00132
00133 auto iter = thread_affinity.find(key);
00134 if (iter != thread_affinity.end())
00135 delete iter->second;
00136 thread_affinity[key] = cpuset;
00137 }
00138 else
00139 ParseWarning(WARN_CONF, "This platform does not support setting thread affinity.\n");
00140 }
00141
00142 void ThreadConfig::implement_thread_affinity(SThreadType type, unsigned id)
00143 {
00144 if (!topology_support->cpubind->set_thisthread_cpubind)
00145 return;
00146
00147 TypeIdPair key { type, id };
00148 hwloc_cpuset_t current_cpuset, desired_cpuset;
00149 char* s;
00150
00151 auto iter = thread_affinity.find(key);
00152 if (iter != thread_affinity.end())
00153 desired_cpuset = iter->second->cpuset;
00154 else
00155 desired_cpuset = process_cpuset;
00156 hwloc_bitmap_list_asprintf(&s, desired_cpuset);
00157
00158 current_cpuset = hwloc_bitmap_alloc();
00159 hwloc_get_cpubind(topology, current_cpuset, HWLOC_CPUBIND_THREAD);
00160 if (!hwloc_bitmap_isequal(current_cpuset, desired_cpuset))
00161 LogMessage("Binding thread %u (type %u) to %s.\n", id, type, s);
00162 hwloc_bitmap_free(current_cpuset);
00163
00164 if (hwloc_set_cpubind(topology, desired_cpuset, HWLOC_CPUBIND_THREAD))
00165 {
00166 FatalError("Failed to pin thread %u (type %u) to %s: %s (%d)\n",
00167 id, type, s, get_error(errno), errno);
00168 }
00169
00170 free(s);
00171 }
00172
00173
00174 // -----------------------------------------------------------------------------
00175 // unit tests
00176 // -----------------------------------------------------------------------------
00177
00178 #ifdef UNIT_TEST
00179
00180 TEST_CASE("Parse cpuset string negative test", "[ThreadConfig]")
00181 {
00182 CpuSet* cpuset = ThreadConfig::validate_cpuset_string("totally a bad cpuset string");
00183 CHECK(cpuset == nullptr);
00184 }
00185
00186 TEST_CASE("Parse cpuset string positive test", "[ThreadConfig]")
00187 {
00188 char* s;
00189 hwloc_bitmap_list_asprintf(&s, process_cpuset);
00190 CpuSet* cpuset = ThreadConfig::validate_cpuset_string(s);
00191 free(s);
00192 CHECK(cpuset != nullptr);
00193 ThreadConfig::destroy_cpuset(cpuset);
00194 }
00195
00196 TEST_CASE("Set and check max packet threads", "[ThreadConfig]")
00197 {
00198 CHECK(ThreadConfig::get_instance_max() == 1);
00199 unsigned new_max = hwloc_bitmap_weight(process_cpuset) * 2;
00200 ThreadConfig::set_instance_max(new_max);
00201 CHECK(ThreadConfig::get_instance_max() == new_max);
00202 ThreadConfig::set_instance_max(0);
00203 CHECK(ThreadConfig::get_instance_max() == hwloc_bitmap_weight(process_cpuset));
00204 }
00205
00206 TEST_CASE("Set and implement thread affinity", "[ThreadConfig]")
00207 {
00208 if (topology_support->cpubind->set_thisthread_cpubind)
00209 {
00210 CpuSet* cpuset = new CpuSet(hwloc_bitmap_dup(process_cpuset));
00211 CpuSet* cpuset2 = new CpuSet(hwloc_bitmap_dup(process_cpuset));
00212 ThreadConfig tc;
00213
00214 hwloc_bitmap_singlify(cpuset->cpuset);
00215 tc.set_thread_affinity(STHREAD_TYPE_PACKET, 0, cpuset2);
00216 tc.set_thread_affinity(STHREAD_TYPE_PACKET, 0, cpuset);
00217 tc.implement_thread_affinity(STHREAD_TYPE_PACKET, 0);
00218
00219 hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc();
00220 hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD);
00221 CHECK(hwloc_bitmap_isequal(thread_cpuset, cpuset->cpuset));
00222
00223 tc.implement_thread_affinity(STHREAD_TYPE_MAIN, 0);
00224 hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD);
00225 CHECK(hwloc_bitmap_isequal(thread_cpuset, process_cpuset));
00226
00227 hwloc_bitmap_free(thread_cpuset);
00228 }
00229 }
00230
00231 #endif
END OF CODE