2 #include "libbenchmark_porting_abstraction_layer_internal.h"
8 /****************************************************************************/
9 #if( defined _WIN32 && !defined KERNEL_MODE && NTDDI_VERSION >= NTDDI_WINXPSP3 && NTDDI_VERSION < NTDDI_WIN7 )
11 #ifdef LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
12 #error More than one porting abstraction layer matches current platform in "libbenchmark_porting_abstraction_layer_populate_topology.c".
15 #define LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
17 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms, struct libbenchmark_topology_node_state *tns, lfds710_pal_uint_t bitmask );
19 int libbenchmark_porting_abstraction_layer_populate_topology( struct libbenchmark_topology_state *ts,
20 struct libshared_memory_state *ms )
30 enum libbenchmark_topology_node_cache_type
31 processor_cache_type_to_libbenchmark_topology_node_cache_type[3] =
33 LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_UNIFIED, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_INSTRUCTION, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_DATA
39 struct libbenchmark_topology_node_state
42 SYSTEM_LOGICAL_PROCESSOR_INFORMATION
48 LFDS710_PAL_ASSERT( ts != NULL );
49 LFDS710_PAL_ASSERT( ms != NULL );
51 // TRD : obtain information from the OS
52 brv = GetLogicalProcessorInformation( slpi, &slpi_length );
53 slpi = libshared_memory_alloc_from_most_free_space_node( ms, slpi_length, sizeof(lfds710_pal_uint_t) );
54 brv = GetLogicalProcessorInformation( slpi, &slpi_length );
55 number_slpi = slpi_length / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
57 /* TRD : we loop twice over the topology information
58 first time we form up the system node
60 second time, we do everything else
63 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
65 for( loop = 0 ; loop < number_slpi ; loop++ )
66 if( (slpi+loop)->Relationship == RelationNumaNode )
67 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) (slpi+loop)->ProcessorMask );
69 libbenchmark_misc_pal_helper_add_system_node_to_topology_tree( ts, tns );
71 for( loop = 0 ; loop < number_slpi ; loop++ )
73 if( (slpi+loop)->Relationship == RelationNumaNode )
75 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
76 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) ((slpi+loop)->ProcessorMask) );
77 libbenchmark_misc_pal_helper_add_numa_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) (slpi+loop)->NumaNode.NodeNumber );
79 // TRD : add each LP as an individual LP node
80 for( mask = 1 ; mask != 0 ; mask <<= 1 )
81 if( ((slpi+loop)->ProcessorMask & mask) == mask )
82 libbenchmark_misc_pal_helper_add_logical_processor_node_to_topology_tree( ts, ms, (lfds710_pal_uint_t) ((slpi+loop)->ProcessorMask & mask), LOWERED, 0 );
85 if( (slpi+loop)->Relationship == RelationProcessorPackage )
87 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
88 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) ((slpi+loop)->ProcessorMask) );
89 libbenchmark_misc_pal_helper_add_socket_node_to_topology_tree( ts, tns );
92 if( (slpi+loop)->Relationship == RelationProcessorCore )
94 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
95 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) ((slpi+loop)->ProcessorMask) );
96 libbenchmark_misc_pal_helper_add_physical_processor_node_to_topology_tree( ts, tns );
99 if( (slpi+loop)->Relationship == RelationCache )
101 if( (slpi+loop)->Cache.Type == CacheUnified or (slpi+loop)->Cache.Type == CacheInstruction or (slpi+loop)->Cache.Type == CacheData )
103 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
104 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) (slpi+loop)->ProcessorMask );
105 libbenchmark_misc_pal_helper_add_cache_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) (slpi+loop)->Cache.Level, processor_cache_type_to_libbenchmark_topology_node_cache_type[(slpi+loop)->Cache.Type] );
113 /****************************************************************************/
114 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms,
115 struct libbenchmark_topology_node_state *tns,
116 lfds710_pal_uint_t bitmask )
119 logical_processor_number = 1;
121 struct libbenchmark_topology_node_state
124 LFDS710_PAL_ASSERT( ms != NULL );
125 LFDS710_PAL_ASSERT( tns != NULL );
126 // TRD : bitmask can be any value in its range
128 /* TRD : iterate over the bits in the bitmask
133 while( bitmask != 0 )
136 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, logical_processor_number, LOWERED, 0 );
139 logical_processor_number++;
151 /****************************************************************************/
152 #if( defined _WIN32 && !defined KERNEL_MODE && NTDDI_VERSION >= NTDDI_WIN7 )
154 #ifdef LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
155 #error More than one porting abstraction layer matches current platform in "libbenchmark_porting_abstraction_layer_populate_topology.c".
158 #define LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
160 static int numa_node_id_to_numa_node_id_compare_function( void const *new_key, void const *existing_key );
161 static void nna_cleanup( struct lfds710_btree_au_state *baus, struct lfds710_btree_au_element *baue );
162 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms, struct libbenchmark_topology_node_state *tns, lfds710_pal_uint_t windows_processor_group_number, lfds710_pal_uint_t bitmask );
164 int libbenchmark_porting_abstraction_layer_populate_topology( struct libbenchmark_topology_state *ts,
165 struct libshared_memory_state *ms )
176 enum libbenchmark_topology_node_cache_type
177 processor_cache_type_to_libbenchmark_topology_node_cache_type[3] =
179 LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_UNIFIED, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_INSTRUCTION, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_DATA
190 logical_processor_number;
192 struct lfds710_btree_au_element
195 struct lfds710_btree_au_state
198 struct libbenchmark_topology_node_state
201 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX
203 *slpie_buffer = NULL;
205 LFDS710_PAL_ASSERT( ts != NULL );
206 LFDS710_PAL_ASSERT( ms != NULL );
208 // TRD : obtain information from the OS
209 brv = GetLogicalProcessorInformationEx( RelationAll, slpie_buffer, &slpie_length );
210 slpie_buffer = libshared_memory_alloc_from_most_free_space_node( ms, slpie_length, sizeof(lfds710_pal_uint_t) );
211 brv = GetLogicalProcessorInformationEx( RelationAll, slpie_buffer, &slpie_length );
213 /* TRD : this API from MS is absolutely bloody appalling
214 staggeringly and completely needlessly complex and inadequately documented
215 I think I've found at least one design flaw
216 and I'm inferring from the C structures a good deal of what's presumably going on
217 where the docs just don't say
219 (addendum - I've just found another huge fucking issue which has wasted two fucking days of my time
220 the original non-Ex() API returns an actual C array, where the elements are structs, which contain
221 a union, but in C the struct is sized to the max size of the union, so you can iterate over the array
223 the NEW version, in the docs still says "array", but it actually returns a PACKED "array" (not an
224 array, because you can't iterate over it) where the each element now has a Size member - you need
225 to move your pointer by the number of bytes in Size - this is NOT in the docs, there is NO example
226 code, and the ONLY WAY YOU CAN GUESS IS TO NOTICE THERE IS A SIZE MEMBER IN THE NEW STRUCT)
228 (for example, just found a one-liner buried in the note on a particular structure
229 returned for a particular node type;
231 "If the PROCESSOR_RELATIONSHIP structure represents a processor core, the GroupCount member is always 1."
233 this *implies* that a physical core is never split across groups
234 this is a very important fact, if you're trying to work with this fucking API
235 but it's not actually SPECIFICALLY STATED
236 it's only implied - and so I do not feel confident in it
237 and the appalling design and appallingly low quality of the docs in general hardly gives me confidence
238 to just go ahead and believe in anything I find written - let alone something which is, offfhand, just
239 implies, buried in some structure notes somewhere
240 this is how it is all the way across this entire bloody API
241 another example is that LPs are not actually returned by the API
242 I'm *inferring* I can get the full list by taking the LP masks presented by the NUMA nodes
243 it's *not* documented - i.e. it's not documented HOW TO GET THE LIST OF LOGICAL PROCESSORS IN THE SYSTEM
246 I'm absolutely 100% certain my use of the API is not fully correct
247 but I have no way to find out
248 MS are bloody idiots - the "processor group" concept is absolutely and utterly crazy
249 and it complicates *everything* by a power of 2
250 rather than simply iterating over the records provided,
251 where just about any entity in the system (NUMA node, processor socket, etc)
252 can have multiple records being returned, I have in fact to iterate over the whole
253 record set, accumulating the multiple records, so I can FINALLY find out the full
254 logical processor set for any given entity, so I can THEN, FINALLY, insert the entity
255 into the toplogy tree
256 i.e. for any given node, you have to fully iterate the list of records provided by
257 the OS, to actually know you know all the LPs for that node
258 there is no single-entity/single-record lookup or relationship
259 MS -> you are bloody idiots; this is appalling, OBVIOUSLY appalling, and whoever
260 designed it, and ESPECIALLY whoever APPROVED It, needs not only to be fired, but SHOT
262 as ever with MS, something that takes a few minutes in Linux takes bloody hours with MS
264 note due to aforementioned design flaw, it is not possible to collect cache information
265 the problem is that if we have a cache which spans multiple processor groups, there will
266 be mutiple records (or I presume there will be - I'm inferring), BUT, looking at the
267 structures, it's not possible to know these are *the same cache*
272 - we need to loop over the full list of records to accumulate the full set of LPs for each NUMA node
273 then we can add the record to tbe btree
275 - really REALLY don't care - with prejudice
276 3. RelationProcessorPackage
277 - bizarrely, actually does the right thing (as far as it can be right in this sorry mess) and contains
278 the full list of group IDs it belongs to, and the full list of LP IDs within each group
279 so we can iterate once over the full set of records and insert this record type directly
280 4. RelationProcessorCore
281 - same as RelationProcessorPackage
283 - seems fubared; provide a single processor group and single mask of LPs, and so if a cache spans
284 multiple processor groups, we'll get multiple records for it - problem is, we've no way of knowing
286 we get away with this with NUMA because each node has an ID
287 the next best thing is going to be record the details of the cache from the structure
288 (level, associativity, etc) and match based on that
292 // TRD : iterate once for system node
293 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
295 while( offset < slpie_length )
297 slpie = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *) ( (char unsigned *) slpie_buffer + offset );
299 offset += slpie->Size;
301 if( slpie->Relationship == RelationNumaNode )
302 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) (slpie->NumaNode.GroupMask.Group), (lfds710_pal_uint_t) (slpie->NumaNode.GroupMask.Mask) );
305 libbenchmark_misc_pal_helper_add_system_node_to_topology_tree( ts, tns );
307 // TRD : iterate again for everything else
308 lfds710_btree_au_init_valid_on_current_logical_core( &nna_tree_state, numa_node_id_to_numa_node_id_compare_function, LFDS710_BTREE_AU_INSERT_RESULT_FAILURE_EXISTING_KEY, ts );
312 while( offset < slpie_length )
314 slpie = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *) ( (char unsigned *) slpie_buffer + offset );
316 offset += slpie->Size;
318 if( slpie->Relationship == RelationNumaNode )
320 /* TRD : now for the first madness - accumulate the NUMA node records
322 first, try to find this node in nna_tree_state
323 if it's there, we use it - it not, we make it and add it, and use it
324 once we've got a node to work with, we add the current list of LPs to that node
327 rv = lfds710_btree_au_get_by_key( &nna_tree_state, NULL, (void *) &slpie->NumaNode.NodeNumber, &baue );
331 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
332 baue = libshared_memory_alloc_from_most_free_space_node( ms, sizeof(struct lfds710_btree_au_element), LFDS710_PAL_ATOMIC_ISOLATION_IN_BYTES );
333 LFDS710_BTREE_AU_SET_KEY_IN_ELEMENT( *baue, (void *) &slpie->NumaNode.NodeNumber );
334 LFDS710_BTREE_AU_SET_VALUE_IN_ELEMENT( *baue, tns );
335 lfds710_btree_au_insert( &nna_tree_state, baue, NULL );
338 // TRD : baue now points at the correct node
339 tns = LFDS710_BTREE_AU_GET_VALUE_FROM_ELEMENT( *baue );
340 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->NumaNode.GroupMask.Group, (lfds710_pal_uint_t) slpie->NumaNode.GroupMask.Mask );
342 // TRD : now all all LPs from this NUMA node to tree
343 logical_processor_number = 0;
344 bitmask = slpie->NumaNode.GroupMask.Mask;
346 while( bitmask != 0 )
349 libbenchmark_misc_pal_helper_add_logical_processor_node_to_topology_tree( ts, ms, logical_processor_number, RAISED, (slpie->NumaNode.GroupMask.Group) );
352 logical_processor_number++;
356 if( slpie->Relationship == RelationGroup )
358 // TRD : we don't care about this - actually, we do care, we really REALLY hate this
361 if( slpie->Relationship == RelationProcessorPackage )
363 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
364 for( subloop = 0 ; subloop < slpie->Processor.GroupCount ; subloop++ )
365 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Group, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Mask );
366 libbenchmark_misc_pal_helper_add_socket_node_to_topology_tree( ts, tns );
369 if( slpie->Relationship == RelationProcessorCore )
371 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
372 for( subloop = 0 ; subloop < slpie->Processor.GroupCount ; subloop++ )
373 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Group, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Mask );
374 libbenchmark_misc_pal_helper_add_physical_processor_node_to_topology_tree( ts, tns );
378 if( slpie->Relationship == RelationCache )
380 if( slpie->Cache.Type == CacheUnified or slpie->Cache.Type == CacheInstruction or slpie->Cache.Type == CacheData )
382 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
383 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->ProcessorMask );
384 libbenchmark_misc_pal_helper_add_cache_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) slpie->Cache.Level, processor_cache_type_to_libbenchmark_topology_node_cache_type[slpie->Cache.Type] );
390 /* TRD : now finally insert the built-up NUMA and cache records
391 we call cleanup() on the accumulator tree - it's safe to re-use the nodes as they're emitted to the cleanup function
392 so we then throw them into the topology_state tree
395 lfds710_btree_au_cleanup( &nna_tree_state, nna_cleanup );
400 /****************************************************************************/
401 static int numa_node_id_to_numa_node_id_compare_function( void const *new_key, void const *existing_key )
407 numa_node_id_existing,
410 LFDS710_PAL_ASSERT( new_key != NULL );
411 LFDS710_PAL_ASSERT( existing_key != NULL );
413 numa_node_id_new = *(DWORD *) new_key;
414 numa_node_id_existing = *(DWORD *) existing_key;
416 if( numa_node_id_new < numa_node_id_existing )
419 if( numa_node_id_new > numa_node_id_existing )
425 /****************************************************************************/
426 static void nna_cleanup( struct lfds710_btree_au_state *baus, struct lfds710_btree_au_element *baue )
431 struct libbenchmark_topology_node_state
434 struct libbenchmark_topology_state
437 LFDS710_PAL_ASSERT( baus != NULL );
438 LFDS710_PAL_ASSERT( baue != NULL );
440 ts = LFDS710_BTREE_AU_GET_USER_STATE_FROM_STATE( *baus );
441 numa_node_id = LFDS710_BTREE_AU_GET_KEY_FROM_ELEMENT( *baue );
442 tns = LFDS710_BTREE_AU_GET_VALUE_FROM_ELEMENT( *baue );
444 libbenchmark_misc_pal_helper_add_numa_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) *numa_node_id );
449 /****************************************************************************/
450 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms,
451 struct libbenchmark_topology_node_state *tns,
452 lfds710_pal_uint_t windows_processor_group_number,
453 lfds710_pal_uint_t bitmask )
456 logical_processor_number = 0;
458 LFDS710_PAL_ASSERT( ms != NULL );
459 LFDS710_PAL_ASSERT( tns != NULL );
460 // TRD : windows_processor_group_number can be any value in its range
461 // TRD : bitmask can be any value in its range
463 /* TRD : iterate over the bits in the bitmask
468 while( bitmask != 0 )
471 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, logical_processor_number, RAISED, windows_processor_group_number );
474 logical_processor_number++;
486 /****************************************************************************/
487 #if( defined _WIN32 && defined KERNEL_MODE && NTDDI_VERSION >= NTDDI_WINXP && NTDDI_VERSION < NTDDI_WIN7 )
489 #ifdef LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
490 #error More than one porting abstraction layer matches current platform in "libbenchmark_porting_abstraction_layer_populate_topology.c".
493 #define LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
495 int libbenchmark_porting_abstraction_layer_populate_topology( struct libbenchmark_topology_state *ts,
496 struct libshared_memory_state *ms )
501 struct libbenchmark_topology_node_state
504 LFDS710_PAL_ASSERT( ts != NULL );
505 LFDS710_PAL_ASSERT( ms != NULL );
507 /* TRD : prior to Windows 7 there is no way to enumerate CPU topology
508 all that is available is a count of the number of logical cores, KeNumberProcessors
509 this is in fact only available *up to Vista SP1*... Windows 7 provides full functionality to get topology,
510 so it's not clear what should be done on Vista SP1...
512 as such to get the topology actually right, the user has to hardcode it
514 the best general solution seems to be to take the number of logical cores
515 assumes they're all on one processor and there's one NUMA node
518 // TRD : create the system node, populate and insert
519 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
520 for( loop = 0 ; loop < KeNumberProcessors ; loop++ )
521 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, loop, LOWERED, 0 );
522 libbenchmark_misc_pal_helper_add_system_node_to_topology_tree( ts, tns );
524 // TRD : create the NUMA node, populate and insert
525 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
526 for( loop = 0 ; loop < KeNumberProcessors ; loop++ )
527 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, loop, LOWERED, 0 );
528 libbenchmark_misc_pal_helper_add_numa_node_to_topology_tree( ts, tns, 0 );
530 // TRD : create the socket node, populate and insert
531 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
532 for( loop = 0 ; loop < KeNumberProcessors ; loop++ )
533 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, loop, LOWERED, 0 );
534 libbenchmark_misc_pal_helper_add_socket_node_to_topology_tree( ts, tns );
536 // TRD : create the physical processor node, populate and insert
537 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
538 for( loop = 0 ; loop < KeNumberProcessors ; loop++ )
539 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, loop, LOWERED, 0 );
540 libbenchmark_misc_pal_helper_add_physical_processor_node_to_topology_tree( ts, tns );
542 // TRD : create the logical processor nodes, populate and insert
543 for( loop = 0 ; loop < KeNumberProcessors ; loop++ )
544 libbenchmark_misc_pal_helper_add_logical_processor_node_to_topology_tree( ts, ms, loop, LOWERED, 0 );
555 /****************************************************************************/
556 #if( defined _WIN32 && defined KERNEL_MODE && NTDDI_VERSION >= NTDDI_WIN7 )
558 #ifdef LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
559 #error More than one porting abstraction layer matches current platform in "libbenchmark_porting_abstraction_layer_populate_topology.c".
562 #define LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
564 static int numa_node_id_to_numa_node_id_compare_function( void const *new_key, void const *existing_key );
565 static void nna_cleanup( struct lfds710_btree_au_state *baus, struct lfds710_btree_au_element *baue );
566 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms, struct libbenchmark_topology_node_state *tns, lfds710_pal_uint_t windows_processor_group_number, lfds710_pal_uint_t bitmask );
568 int libbenchmark_porting_abstraction_layer_populate_topology( struct libbenchmark_topology_state *ts,
569 struct libshared_memory_state *ms )
572 enum libbenchmark_topology_node_cache_type
573 processor_cache_type_to_libbenchmark_topology_node_cache_type[3] =
575 LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_UNIFIED, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_INSTRUCTION, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_DATA
586 logical_processor_number;
591 struct lfds710_btree_au_element
594 struct lfds710_btree_au_state
597 struct libbenchmark_topology_node_state
600 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX
602 *slpie_buffer = NULL;
609 LFDS710_PAL_ASSERT( ts != NULL );
610 LFDS710_PAL_ASSERT( ms != NULL );
612 // TRD : obtain information from the OS
613 brv = KeQueryLogicalProcessorRelationship( NULL, RelationAll, slpie_buffer, &slpie_length );
614 slpie_buffer = libshared_memory_alloc_from_most_free_space_node( ms, slpie_length, sizeof(lfds710_pal_uint_t) );
615 brv = KeQueryLogicalProcessorRelationship( NULL, RelationAll, slpie_buffer, &slpie_length );
617 /* TRD : this API from MS is absolutely bloody appalling
618 staggeringly and completely needlessly complex and inadequately documented
619 I think I've found at least one design flaw
620 and I'm inferring from the C structures a good deal of what's presumably going on
621 where the docs just don't say
623 (addendum - I've just found another huge fucking issue which has wasted two fucking days of my time
624 the original non-Ex() API returns an actual C array, where the elements are structs, which contain
625 a union, but in C the struct is sized to the max size of the union, so you can iterate over the array
627 the NEW version, in the docs still says "array", but it actually returns a PACKED "array" (not an
628 array, because you can't iterate over it) where the each element now has a Size member - you need
629 to move your pointer by the number of bytes in Size - this is NOT in the docs, there is NO example
630 code, and the ONLY WAY YOU CAN GUESS IS TO NOTICE THERE IS A SIZE MEMBER IN THE NEW STRUCT)
632 (for example, just found a one-liner buried in the note on a particular structure
633 returned for a particular node type;
635 "If the PROCESSOR_RELATIONSHIP structure represents a processor core, the GroupCount member is always 1."
637 this *implies* that a physical core is never split across groups
638 this is a very important fact, if you're trying to work with this fucking API
639 but it's not actually SPECIFICALLY STATED
640 it's only implied - and so I do not feel confident in it
641 and the appalling design and appallingly low quality of the docs in general hardly gives me confidence
642 to just go ahead and believe in anything I find written - let alone something which is, offfhand, just
643 implies, buried in some structure notes somewhere
644 this is how it is all the way across this entire bloody API
645 another example is that LPs are not actually returned by the API
646 I'm *inferring* I can get the full list by taking the LP masks presented by the NUMA nodes
647 it's *not* documented - i.e. it's not documented HOW TO GET THE LIST OF LOGICAL PROCESSORS IN THE SYSTEM
650 I'm absolutely 100% certain my use of the API is not fully correct
651 but I have no way to find out
652 MS are bloody idiots - the "processor group" concept is absolutely and utterly crazy
653 and it complicates *everything* by a power of 2
654 rather than simply iterating over the records provided,
655 where just about any entity in the system (NUMA node, processor socket, etc)
656 can have multiple records being returned, I have in fact to iterate over the whole
657 record set, accumulating the multiple records, so I can FINALLY find out the full
658 logical processor set for any given entity, so I can THEN, FINALLY, insert the entity
659 into the toplogy tree
660 i.e. for any given node, you have to fully iterate the list of records provided by
661 the OS, to actually know you know all the LPs for that node
662 there is no single-entity/single-record lookup or relationship
663 MS -> you are bloody idiots; this is appalling, OBVIOUSLY appalling, and whoever
664 designed it, and ESPECIALLY whoever APPROVED It, needs not only to be fired, but SHOT
666 as ever with MS, something that takes a few minutes in Linux takes bloody hours with MS
668 note due to aforementioned design flaw, it is not possible to collect cache information
669 the problem is that if we have a cache which spans multiple processor groups, there will
670 be mutiple records (or I presume there will be - I'm inferring), BUT, looking at the
671 structures, it's not possible to know these are *the same cache*
676 - we need to loop over the full list of records to accumulate the full set of LPs for each NUMA node
677 then we can add the record to tbe btree
679 - really REALLY don't care - with prejudice
680 3. RelationProcessorPackage
681 - bizarrely, actually does the right thing (as far as it can be right in this sorry mess) and contains
682 the full list of group IDs it belongs to, and the full list of LP IDs within each group
683 so we can iterate once over the full set of records and insert this record type directly
684 4. RelationProcessorCore
685 - same as RelationProcessorPackage
687 - seems fubared; provide a single processor group and single mask of LPs, and so if a cache spans
688 multiple processor groups, we'll get multiple records for it - problem is, we've no way of knowing
690 we get away with this with NUMA because each node has an ID
691 the next best thing is going to be record the details of the cache from the structure
692 (level, associativity, etc) and match based on that
696 // TRD : iterate once for system node
697 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
699 while( offset < slpie_length )
701 slpie = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *) ( (char unsigned *) slpie_buffer + offset );
703 offset += slpie->Size;
705 if( slpie->Relationship == RelationNumaNode )
706 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) (slpie->NumaNode.GroupMask.Group), (lfds710_pal_uint_t) (slpie->NumaNode.GroupMask.Mask) );
709 libbenchmark_misc_pal_helper_add_system_node_to_topology_tree( ts, tns );
711 // TRD : iterate again for everything else
712 lfds710_btree_au_init_valid_on_current_logical_core( &nna_tree_state, numa_node_id_to_numa_node_id_compare_function, LFDS710_BTREE_AU_INSERT_RESULT_FAILURE_EXISTING_KEY, ts );
716 while( offset < slpie_length )
718 slpie = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *) ( (char unsigned *) slpie_buffer + offset );
720 offset += slpie->Size;
722 if( slpie->Relationship == RelationNumaNode )
724 /* TRD : now for the first madness - accumulate the NUMA node records
726 first, try to find this node in nna_tree_state
727 if it's there, we use it - it not, we make it and add it, and use it
728 once we've got a node to work with, we add the current list of LPs to that node
731 rv = lfds710_btree_au_get_by_key( &nna_tree_state, NULL, (void *) &slpie->NumaNode.NodeNumber, &baue );
735 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
736 baue = libshared_memory_alloc_from_most_free_space_node( ms, sizeof(struct lfds710_btree_au_element), LFDS710_PAL_ATOMIC_ISOLATION_IN_BYTES );
737 LFDS710_BTREE_AU_SET_KEY_IN_ELEMENT( *baue, (void *) &slpie->NumaNode.NodeNumber );
738 LFDS710_BTREE_AU_SET_VALUE_IN_ELEMENT( *baue, tns );
739 lfds710_btree_au_insert( &nna_tree_state, baue, NULL );
742 // TRD : baue now points at the correct node
743 tns = LFDS710_BTREE_AU_GET_VALUE_FROM_ELEMENT( *baue );
744 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->NumaNode.GroupMask.Group, (lfds710_pal_uint_t) slpie->NumaNode.GroupMask.Mask );
746 // TRD : now all all LPs from this NUMA node to tree
747 logical_processor_number = 0;
748 bitmask = slpie->NumaNode.GroupMask.Mask;
750 while( bitmask != 0 )
753 libbenchmark_misc_pal_helper_add_logical_processor_node_to_topology_tree( ts, ms, logical_processor_number, RAISED, (slpie->NumaNode.GroupMask.Group) );
756 logical_processor_number++;
760 if( slpie->Relationship == RelationGroup )
762 // TRD : we don't care about this - actually, we do care, we really REALLY hate this
765 if( slpie->Relationship == RelationProcessorPackage )
767 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
768 for( subloop = 0 ; subloop < slpie->Processor.GroupCount ; subloop++ )
769 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Group, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Mask );
770 libbenchmark_misc_pal_helper_add_socket_node_to_topology_tree( ts, tns );
773 if( slpie->Relationship == RelationProcessorCore )
775 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
776 for( subloop = 0 ; subloop < slpie->Processor.GroupCount ; subloop++ )
777 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Group, (lfds710_pal_uint_t) slpie->Processor.GroupMask[subloop].Mask );
778 libbenchmark_misc_pal_helper_add_physical_processor_node_to_topology_tree( ts, tns );
782 if( slpie->Relationship == RelationCache )
784 if( slpie->Cache.Type == CacheUnified or slpie->Cache.Type == CacheInstruction or slpie->Cache.Type == CacheData )
786 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
787 internal_populate_logical_processor_array_from_bitmask( ms, tns, (lfds710_pal_uint_t) slpie->ProcessorMask );
788 libbenchmark_misc_pal_helper_add_cache_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) slpie->Cache.Level, processor_cache_type_to_libbenchmark_topology_node_cache_type[slpie->Cache.Type] );
794 /* TRD : now finally insert the built-up NUMA and cache records
795 we call cleanup() on the accumulator tree - it's safe to re-use the nodes as they're emitted to the cleanup function
796 so we then throw them into the topology_state tree
799 lfds710_btree_au_cleanup( &nna_tree_state, nna_cleanup );
804 /****************************************************************************/
805 static int numa_node_id_to_numa_node_id_compare_function( void const *new_key, void const *existing_key )
811 numa_node_id_existing,
814 LFDS710_PAL_ASSERT( new_key != NULL );
815 LFDS710_PAL_ASSERT( existing_key != NULL );
817 numa_node_id_new = *(ULONG *) new_key;
818 numa_node_id_existing = *(ULONG *) existing_key;
820 if( numa_node_id_new < numa_node_id_existing )
823 if( numa_node_id_new > numa_node_id_existing )
829 /****************************************************************************/
830 static void nna_cleanup( struct lfds710_btree_au_state *baus, struct lfds710_btree_au_element *baue )
835 struct libbenchmark_topology_node_state
838 struct libbenchmark_topology_state
841 LFDS710_PAL_ASSERT( baus != NULL );
842 LFDS710_PAL_ASSERT( baue != NULL );
844 ts = LFDS710_BTREE_AU_GET_USER_STATE_FROM_STATE( *baus );
845 numa_node_id = LFDS710_BTREE_AU_GET_KEY_FROM_ELEMENT( *baue );
846 tns = LFDS710_BTREE_AU_GET_VALUE_FROM_ELEMENT( *baue );
848 libbenchmark_misc_pal_helper_add_numa_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) *numa_node_id );
853 /****************************************************************************/
854 static void internal_populate_logical_processor_array_from_bitmask( struct libshared_memory_state *ms,
855 struct libbenchmark_topology_node_state *tns,
856 lfds710_pal_uint_t windows_processor_group_number,
857 lfds710_pal_uint_t bitmask )
860 logical_processor_number = 0;
862 LFDS710_PAL_ASSERT( ms != NULL );
863 LFDS710_PAL_ASSERT( tns != NULL );
864 // TRD : windows_processor_group_number can be any value in its range
865 // TRD : bitmask can be any value in its range
867 /* TRD : iterate over the bits in the bitmask
872 while( bitmask != 0 )
875 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, logical_processor_number, RAISED, windows_processor_group_number );
878 logical_processor_number++;
890 /****************************************************************************/
891 #if( defined __linux__ && !defined KERNEL_MODE && defined __STDC__ && __STDC_HOSTED__ == 1 )
893 #ifdef LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
894 #error More than one porting abstraction layer matches current platform in "libbenchmark_porting_abstraction_layer_populate_topology.c".
897 #define LIBBENCHMARK_PAL_POPULATE_TOPOLOGY
899 static void internal_populate_logical_processor_array_from_path_to_csv_hex( struct libshared_memory_state *ms,
900 struct libbenchmark_topology_node_state *tns,
901 char *path_to_csv_hex );
902 static int internal_verify_paths( lfds710_pal_uint_t number_paths, ... );
903 static void internal_read_string_from_path( char *path, char *string );
905 /****************************************************************************/
906 int libbenchmark_porting_abstraction_layer_populate_topology( struct libbenchmark_topology_state *ts,
907 struct libshared_memory_state *ms )
911 thread_siblings_path[128],
912 core_siblings_path[128],
913 cache_level_path[128],
914 cache_type_path[128],
915 shared_cpu_map_path[128],
916 cache_level_string[16],
917 cache_type_string[16];
921 cache_type_string_to_type_enum_lookup[NUMBER_UPPERCASE_LETTERS_IN_LATIN_ALPHABET] =
923 -1, -1, -1, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_DATA, -1, -1, -1, -1, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_INSTRUCTION, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, LIBBENCHMARK_TOPOLOGY_NODE_CACHE_TYPE_UNIFIED, -1, -1, -1, -1
926 int long long unsigned
936 struct libbenchmark_topology_iterate_state
939 struct libbenchmark_topology_node_state
943 LFDS710_PAL_ASSERT( ts != NULL );
944 LFDS710_PAL_ASSERT( ms != NULL );
946 sprintf( numa_node_path, "/sys/devices/system/node/node%llu/cpumap", (int long long unsigned) numa_node );
948 while( internal_verify_paths(1, numa_node_path) )
950 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
951 internal_populate_logical_processor_array_from_path_to_csv_hex( ms, tns, numa_node_path );
952 libbenchmark_misc_pal_helper_add_numa_node_to_topology_tree( ts, tns, (lfds710_pal_uint_t) numa_node );
953 sprintf( numa_node_path, "/sys/devices/system/node/node%llu/cpumap", (int long long unsigned) (++numa_node) );
956 sprintf( thread_siblings_path, "/sys/devices/system/cpu/cpu%llu/topology/thread_siblings", (int long long unsigned) cpu_number );
957 sprintf( core_siblings_path, "/sys/devices/system/cpu/cpu%llu/topology/core_siblings", (int long long unsigned) cpu_number );
959 while( internal_verify_paths(2, core_siblings_path, thread_siblings_path) )
961 libbenchmark_misc_pal_helper_add_logical_processor_node_to_topology_tree( ts, ms, cpu_number, LOWERED, 0 );
963 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
964 internal_populate_logical_processor_array_from_path_to_csv_hex( ms, tns, thread_siblings_path );
965 libbenchmark_misc_pal_helper_add_physical_processor_node_to_topology_tree( ts, tns );
967 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
968 internal_populate_logical_processor_array_from_path_to_csv_hex( ms, tns, core_siblings_path );
969 libbenchmark_misc_pal_helper_add_socket_node_to_topology_tree( ts, tns );
973 sprintf( cache_level_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/level", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
974 sprintf( cache_type_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/type", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
975 sprintf( shared_cpu_map_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/shared_cpu_map", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
977 while( internal_verify_paths(3, cache_level_path, cache_type_path, shared_cpu_map_path) )
979 internal_read_string_from_path( cache_level_path, cache_level_string );
980 sscanf( cache_level_string, "%llx", &level_temp );
981 level = (lfds710_pal_uint_t) level_temp;
983 internal_read_string_from_path( cache_type_path, cache_type_string );
984 type = cache_type_string_to_type_enum_lookup[(int)(*cache_type_string - 'A')];
986 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
987 internal_populate_logical_processor_array_from_path_to_csv_hex( ms, tns, shared_cpu_map_path );
988 libbenchmark_misc_pal_helper_add_cache_node_to_topology_tree( ts, tns, level, type );
992 sprintf( cache_level_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/level", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
993 sprintf( cache_type_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/type", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
994 sprintf( shared_cpu_map_path, "/sys/devices/system/cpu/cpu%llu/cache/index%llu/shared_cpu_map", (int long long unsigned) cpu_number, (int long long unsigned) index_number );
999 sprintf( thread_siblings_path, "/sys/devices/system/cpu/cpu%llu/topology/thread_siblings", (int long long unsigned) cpu_number );
1000 sprintf( core_siblings_path, "/sys/devices/system/cpu/cpu%llu/topology/core_siblings", (int long long unsigned) cpu_number );
1003 // TRD : now make and populate the notional system node
1004 libbenchmark_misc_pal_helper_new_topology_node( &tns, ms );
1005 libbenchmark_topology_iterate_init( &tis, LIBBENCHMARK_TOPOLOGY_NODE_TYPE_LOGICAL_PROCESSOR );
1006 while( libbenchmark_topology_iterate(ts, &tis, &tns_lp) )
1007 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, LIBBENCHMARK_TOPOLOGY_NODE_GET_LOGICAL_PROCESSOR_NUMBER(*tns_lp), LOWERED, 0 );
1008 libbenchmark_misc_pal_helper_add_system_node_to_topology_tree( ts, tns );
1013 /****************************************************************************/
1014 void libbenchmark_porting_abstraction_layer_topology_node_cleanup( struct libbenchmark_topology_node_state *tns )
1016 LFDS710_PAL_ASSERT( tns != NULL );
1018 lfds710_list_aso_cleanup( &tns->logical_processor_children, NULL );
1023 /****************************************************************************/
1024 static void internal_populate_logical_processor_array_from_path_to_csv_hex( struct libshared_memory_state *ms,
1025 struct libbenchmark_topology_node_state *tns,
1026 char *path_to_csv_hex )
1039 logical_processor_foursome,
1040 logical_processor_number = 0,
1046 LFDS710_PAL_ASSERT( ms != NULL );
1047 LFDS710_PAL_ASSERT( tns != NULL );
1048 LFDS710_PAL_ASSERT( path_to_csv_hex != NULL );
1050 /* TRD : we're passed a format string and args, which comprise the path
1051 form up the string, open the file, read the string, parse the string
1052 the string consists of 32-bit bitmasks in hex separated by commas
1053 no leading or trailing commas
1056 diskfile = fopen( path_to_csv_hex, "r" );
1057 setbuf( diskfile, diskbuffer );
1058 fgets( string, 1024, diskfile );
1061 while( string[length++] != '\0' );
1065 for( loop = ((int)length)-1 ; loop > -1 ; loop-- )
1067 if( string[loop] == ',' )
1070 sscanf( &string[loop], "%1x", &logical_processor_foursome );
1072 for( subloop = 0 ; subloop < 4 ; subloop++ )
1073 if( ( (logical_processor_foursome >> subloop) & 0x1 ) == 0x1 )
1074 libbenchmark_misc_pal_helper_add_logical_processor_to_topology_node( tns, ms, logical_processor_number + subloop, LOWERED, 0 );
1076 logical_processor_number += 4;
1082 /****************************************************************************/
1083 static int internal_verify_paths( lfds710_pal_uint_t number_paths, ... )
1097 // TRD : number_paths can be any value in its range
1099 va_start( va, number_paths );
1101 while( rv == 1 and count++ < number_paths )
1102 if( NULL == (diskfile = fopen(va_arg(va,char *), "r")) )
1112 /****************************************************************************/
1113 static void internal_read_string_from_path( char *path, char *string )
1121 LFDS710_PAL_ASSERT( path != NULL );
1122 LFDS710_PAL_ASSERT( string != NULL );
1124 diskfile = fopen( path, "r" );
1125 setbuf( diskfile, diskbuffer );
1126 fscanf( diskfile, "%s", string );
1138 /****************************************************************************/
1139 #if( !defined LIBBENCHMARK_PAL_POPULATE_TOPOLOGY )
1141 #error No matching porting abstraction layer in "libbenchmark_porting_abstraction_layer_populate_topology.c".