#include #include #include #include #include #include #if defined(__ANDROID__) #include #endif #include #include #include #include #include static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) { return (bitfield & mask) == mask; } /* * Assigns logical processors to clusters of cores using heuristic based on the typical configuration of clusters for * 5, 6, 8, and 10 cores: * - 5 cores (ARM32 Android only): 2 clusters of 4+1 cores * - 6 cores: 2 clusters of 4+2 cores * - 8 cores: 2 clusters of 4+4 cores * - 10 cores: 3 clusters of 4+4+2 cores * * The function must be called after parsing OS-provided information on core clusters. * Its purpose is to detect clusters of cores when OS-provided information is lacking or incomplete, i.e. * - Linux kernel is not configured to report information in sysfs topology leaf. * - Linux kernel reports topology information only for online cores, and only cores on one cluster are online, e.g.: * - Exynos 8890 has 8 cores in 4+4 clusters, but only the first cluster of 4 cores is reported, and cluster * configuration of logical processors 4-7 is not reported (all remaining processors 4-7 form cluster 1) * - MT6797 has 10 cores in 4+4+2, but only the first cluster of 4 cores is reported, and cluster configuration * of logical processors 4-9 is not reported (processors 4-7 form cluster 1, and processors 8-9 form cluster 2). * * Heuristic assignment of processors to the above pre-defined clusters fails if such assignment would contradict * information provided by the operating system: * - Any of the OS-reported processor clusters is different than the corresponding heuristic cluster. * - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have known and different * minimum/maximum frequency. * - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have known and different * MIDR components. * * If the heuristic assignment of processors to clusters of cores fails, all processors' clusters are unchanged. * * @param usable_processors - number of processors in the @p processors array with CPUINFO_LINUX_FLAG_VALID flags. * @param max_processors - number of elements in the @p processors array. * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, minimum/maximum * frequency, MIDR information, and core cluster (package siblings list) information. * * @retval true if the heuristic successfully assigned all processors into clusters of cores. * @retval false if known details about processors contradict the heuristic configuration of core clusters. */ bool cpuinfo_arm_linux_detect_core_clusters_by_heuristic( uint32_t usable_processors, uint32_t max_processors, struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) { uint32_t cluster_processors[3]; switch (usable_processors) { case 10: cluster_processors[0] = 4; cluster_processors[1] = 4; cluster_processors[2] = 2; break; case 8: cluster_processors[0] = 4; cluster_processors[1] = 4; break; case 6: cluster_processors[0] = 4; cluster_processors[1] = 2; break; #if defined(__ANDROID__) && CPUINFO_ARCH_ARM case 5: /* * The only processor with 5 cores is Leadcore L1860C (ARMv7, mobile), * but this configuration is not too unreasonable for a virtualized ARM server. */ cluster_processors[0] = 4; cluster_processors[1] = 1; break; #endif default: return false; } /* * Assignment of processors to core clusters is done in two passes: * 1. Verify that the clusters proposed by heuristic are compatible with known details about processors. * 2. If verification passed, update core clusters for the processors. */ uint32_t cluster = 0; uint32_t expected_cluster_processors = 0; uint32_t cluster_start, cluster_flags, cluster_midr, cluster_max_frequency, cluster_min_frequency; bool expected_cluster_exists; for (uint32_t i = 0; i < max_processors; i++) { if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { if (expected_cluster_processors == 0) { /* Expect this processor to start a new cluster */ expected_cluster_exists = !!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER); if (expected_cluster_exists) { if (processors[i].package_leader_id != i) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "processor %"PRIu32" is expected to start a new cluster #%"PRIu32" with %"PRIu32" cores, " "but system siblings lists reported it as a sibling of processor %"PRIu32, i, cluster, cluster_processors[cluster], processors[i].package_leader_id); return false; } } else { cluster_flags = 0; } cluster_start = i; expected_cluster_processors = cluster_processors[cluster++]; } else { /* Expect this processor to belong to the same cluster as processor */ if (expected_cluster_exists) { /* * The cluster suggested by the heuristic was already parsed from system siblings lists. * For all processors we expect in the cluster, check that: * - They have pre-assigned cluster from siblings lists (CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER flag). * - They were assigned to the same cluster based on siblings lists * (package_leader_id points to the first processor in the cluster). */ if ((processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) == 0) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "processor %"PRIu32" is expected to belong to the cluster of processor %"PRIu32", " "but system siblings lists did not report it as a sibling of processor %"PRIu32, i, cluster_start, cluster_start); return false; } if (processors[i].package_leader_id != cluster_start) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "processor %"PRIu32" is expected to belong to the cluster of processor %"PRIu32", " "but system siblings lists reported it to belong to the cluster of processor %"PRIu32, i, cluster_start, cluster_start); return false; } } else { /* * The cluster suggest by the heuristic was not parsed from system siblings lists. * For all processors we expect in the cluster, check that: * - They have no pre-assigned cluster from siblings lists. * - If their min/max CPU frequency is known, it is the same. * - If any part of their MIDR (Implementer, Variant, Part, Revision) is known, it is the same. */ if (processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "processor %"PRIu32" is expected to be unassigned to any cluster, " "but system siblings lists reported it to belong to the cluster of processor %"PRIu32, i, processors[i].package_leader_id); return false; } if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { if (cluster_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { if (cluster_min_frequency != processors[i].min_frequency) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "minimum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of its expected cluster (%"PRIu32" KHz)", i, processors[i].min_frequency, cluster_min_frequency); return false; } } else { cluster_min_frequency = processors[i].min_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; } } if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { if (cluster_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { if (cluster_max_frequency != processors[i].max_frequency) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "maximum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of its expected cluster (%"PRIu32" KHz)", i, processors[i].max_frequency, cluster_max_frequency); return false; } } else { cluster_max_frequency = processors[i].max_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK)) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "CPU Implementer of processor %"PRIu32" (0x%02"PRIx32") is different than of its expected cluster (0x%02"PRIx32")", i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr)); return false; } } else { cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIANT_MASK)) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "CPU Variant of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")", i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr)); return false; } } else { cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_PART) { if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MASK)) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "CPU Part of processor %"PRIu32" (0x%03"PRIx32") is different than of its expected cluster (0x%03"PRIx32")", i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr)); return false; } } else { cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_REVISION) { if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVISION_MASK)) { cpuinfo_log_debug( "heuristic detection of core clusters failed: " "CPU Revision of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")", i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr)); return false; } } else { cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; } } } } expected_cluster_processors--; } } /* Verification passed, assign all processors to new clusters */ cluster = 0; expected_cluster_processors = 0; for (uint32_t i = 0; i < max_processors; i++) { if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { if (expected_cluster_processors == 0) { /* Expect this processor to start a new cluster */ cluster_start = i; expected_cluster_processors = cluster_processors[cluster++]; } else { /* Expect this processor to belong to the same cluster as processor */ if (!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) { cpuinfo_log_debug("assigned processor %"PRIu32" to cluster of processor %"PRIu32" based on heuristic", i, cluster_start); } processors[i].package_leader_id = cluster_start; processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; } expected_cluster_processors--; } } return true; } /* * Assigns logical processors to clusters of cores in sequential manner: * - Clusters detected from OS-provided information are unchanged: * - Processors assigned to these clusters stay assigned to the same clusters * - No new processors are added to these clusters * - Processors without pre-assigned cluster are clustered in one sequential scan: * - If known details (min/max frequency, MIDR components) of a processor are compatible with a preceding * processor, without pre-assigned cluster, the processor is assigned to the cluster of the preceding processor. * - If known details (min/max frequency, MIDR components) of a processor are not compatible with a preceding * processor, the processor is assigned to a newly created cluster. * * The function must be called after parsing OS-provided information on core clusters, and usually is called only * if heuristic assignment of processors to clusters (cpuinfo_arm_linux_cluster_processors_by_heuristic) failed. * * Its purpose is to detect clusters of cores when OS-provided information is lacking or incomplete, i.e. * - Linux kernel is not configured to report information in sysfs topology leaf. * - Linux kernel reports topology information only for online cores, and all cores on some of the clusters are offline. * * Sequential assignment of processors to clusters always succeeds, and upon exit, all usable processors in the * @p processors array have cluster information. * * @param max_processors - number of elements in the @p processors array. * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, minimum/maximum * frequency, MIDR information, and core cluster (package siblings list) information. * * @retval true if the heuristic successfully assigned all processors into clusters of cores. * @retval false if known details about processors contradict the heuristic configuration of core clusters. */ void cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan( uint32_t max_processors, struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) { uint32_t cluster_flags = 0; uint32_t cluster_processors = 0; uint32_t cluster_start, cluster_midr, cluster_max_frequency, cluster_min_frequency; for (uint32_t i = 0; i < max_processors; i++) { if ((processors[i].flags & (CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) == CPUINFO_LINUX_FLAG_VALID) { if (cluster_processors == 0) { goto new_cluster; } if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { if (cluster_flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { if (cluster_min_frequency != processors[i].min_frequency) { cpuinfo_log_info( "minimum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of preceding cluster (%"PRIu32" KHz); " "processor %"PRIu32" starts to a new cluster", i, processors[i].min_frequency, cluster_min_frequency, i); goto new_cluster; } } else { cluster_min_frequency = processors[i].min_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; } } if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { if (cluster_flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { if (cluster_max_frequency != processors[i].max_frequency) { cpuinfo_log_debug( "maximum frequency of processor %"PRIu32" (%"PRIu32" KHz) is different than of preceding cluster (%"PRIu32" KHz); " "processor %"PRIu32" starts a new cluster", i, processors[i].max_frequency, cluster_max_frequency, i); goto new_cluster; } } else { cluster_max_frequency = processors[i].max_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK)) { cpuinfo_log_debug( "CPU Implementer of processor %"PRIu32" (0x%02"PRIx32") is different than of preceding cluster (0x%02"PRIx32"); " "processor %"PRIu32" starts to a new cluster", i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr), i); goto new_cluster; } } else { cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIANT_MASK)) { cpuinfo_log_debug( "CPU Variant of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")" "processor %"PRIu32" starts to a new cluster", i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr), i); goto new_cluster; } } else { cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_PART) { if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MASK)) { cpuinfo_log_debug( "CPU Part of processor %"PRIu32" (0x%03"PRIx32") is different than of its expected cluster (0x%03"PRIx32")" "processor %"PRIu32" starts to a new cluster", i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr), i); goto new_cluster; } } else { cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; } } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { if (cluster_flags & CPUINFO_ARM_LINUX_VALID_REVISION) { if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVISION_MASK)) { cpuinfo_log_debug( "CPU Revision of processor %"PRIu32" (0x%"PRIx32") is different than of its expected cluster (0x%"PRIx32")" "processor %"PRIu32" starts to a new cluster", i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr), i); goto new_cluster; } } else { cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; } } /* All checks passed, attach processor to the preceding cluster */ cluster_processors++; processors[i].package_leader_id = cluster_start; processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; cpuinfo_log_debug("assigned processor %"PRIu32" to preceding cluster of processor %"PRIu32, i, cluster_start); continue; new_cluster: /* Create a new cluster starting with processor i */ cluster_start = i; processors[i].package_leader_id = i; processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER; cluster_processors = 1; /* Copy known information from processor to cluster, and set the flags accordingly */ cluster_flags = 0; if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) { cluster_min_frequency = processors[i].min_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY; } if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) { cluster_max_frequency = processors[i].max_frequency; cluster_flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY; } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) { cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_IMPLEMENTER; } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) { cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_VARIANT; } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) { cluster_midr = midr_copy_part(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_PART; } if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) { cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr); cluster_flags |= CPUINFO_ARM_LINUX_VALID_REVISION; } } } } /* * Counts the number of logical processors in each core cluster. * This function should be called after all processors are assigned to core clusters. * * @param max_processors - number of elements in the @p processors array. * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, * and decoded core cluster (package_leader_id) information. * The function expects the value of processors[i].package_processor_count to be zero. * Upon return, processors[i].package_processor_count will contain the number of logical * processors in the respective core cluster. */ void cpuinfo_arm_linux_count_cluster_processors( uint32_t max_processors, struct cpuinfo_arm_linux_processor processors[restrict static max_processors]) { /* First pass: accumulate the number of processors at the group leader's package_processor_count */ for (uint32_t i = 0; i < max_processors; i++) { if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { const uint32_t package_leader_id = processors[i].package_leader_id; processors[package_leader_id].package_processor_count += 1; } } /* Second pass: copy the package_processor_count from the group leader processor */ for (uint32_t i = 0; i < max_processors; i++) { if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { const uint32_t package_leader_id = processors[i].package_leader_id; processors[i].package_processor_count = processors[package_leader_id].package_processor_count; } } }