hw/arm/virt: add cache hierarchy to device tree
Specify which level (core/cluster/socket) caches found at in the CPU topology. Updating cache topology to device tree (spec v0.4). Example: For example, 2 sockets (packages), and 2 clusters, 4 cores and 2 threads created, in aggregate 2*2*4*2 logical cores. In the smp-cache object, cores will have L1d and L1i. However, extending this is not difficult. The clusters will share a unified L2 level cache, and finally sockets will share L3. In this patch, threads will share L1 caches by default, but this can be adjusted if case required. Only three levels of caches are supported. The patch does not allow partial declaration of caches. In other words, the topology level of every cache must be specified if that of any level is. ./qemu-system-aarch64 \ -machine virt,\ smp-cache.0.cache=l1i,smp-cache.0.topology=core,\ smp-cache.1.cache=l1d,smp-cache.1.topology=core,\ smp-cache.2.cache=l2,smp-cache.2.topology=cluster,\ smp-cache.3.cache=l3,smp-cache.3.topology=socket\ -cpu max \ -m 2048 \ -smp sockets=2,clusters=2,cores=4,threads=1 \ -kernel ./Image.gz \ -append "console=ttyAMA0 root=/dev/ram rdinit=/init acpi=force" \ -initrd rootfs.cpio.gz \ -bios ./edk2-aarch64-code.fd \ -nographic For instance, following device tree will be generated for a scenario where we have 2 sockets, 2 clusters, 2 cores and 2 threads, in total 16 PEs. L1i and L1d are private to each thread, and L2 and L3 are shared at socket level as an example. Limitation: SMT cores cannot share L1 cache for now. This problem does not exist in PPTT tables. Co-developed-by: Jonathan Cameron <jonathan.cameron@huawei.com> Signed-off-by: Jonathan Cameron <jonathan.cameron@huawei.com> Signed-off-by: Alireza Sanaee <alireza.sanaee@huawei.com> Reviewed-by: Gustavo Romero <gustavo.romero@linaro.org> Message-id: 20260311160609.358-4-alireza.sanaee@huawei.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Alireza Sanaee committed
Apr 23, 2026 at 10:24 UTC
f669de700111cf9eb540a63c8abbcc630770d4fd
3 files changed
+362
-2
hw/arm/virt.c
+334
-1
@@ -91,6 +91,7 @@
91
#include "hw/virtio/virtio-md-pci.h"
92
#include "hw/virtio/virtio-iommu.h"
93
#include "hw/char/pl011.h"
94
+#include "hw/core/cpu.h"
95
#include "hw/cxl/cxl.h"
96
#include "hw/cxl/cxl_host.h"
97
#include "qemu/guest-random.h"
@@ -281,6 +282,86 @@ static bool ns_el2_virt_timer_present(void)
282
arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu);
283
}
284
285
+void set_cpu_cache(CPUCoreCaches *cpu_cache, enum CacheType cache_type,
286
+ int cache_level, bool is_i_cache0)
287
+{
288
+ int bank_index = ((cache_level - 1) * 2) | is_i_cache0;
289
+ ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(0));
290
+ bool ccidx = cpu_isar_feature(any_ccidx, armcpu);
291
+
292
+ if (ccidx) {
293
+ *cpu_cache = (CPUCoreCaches){
294
+ .linesize = 1 << (FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1,
295
+ CCIDX_LINESIZE) + 4),
296
+ .associativity = FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1,
297
+ CCIDX_ASSOCIATIVITY) + 1,
298
+ .sets = FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1,
299
+ CCIDX_NUMSETS) + 1,
300
+ };
301
+ } else {
302
+ *cpu_cache = (CPUCoreCaches){
303
+ .linesize = 1 << (FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1,
304
+ LINESIZE) + 4),
305
+ .associativity = FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1,
306
+ ASSOCIATIVITY) + 1,
307
+ .sets =
308
+ FIELD_EX64(armcpu->ccsidr[bank_index], CCSIDR_EL1, NUMSETS) + 1,
309
+ };
310
+ }
311
+ cpu_cache->type = cache_type;
312
+ cpu_cache->level = cache_level;
313
+ cpu_cache->size = cpu_cache->associativity *
314
+ cpu_cache->sets *
315
+ cpu_cache->linesize;
316
+
317
+ return;
318
+}
319
+
320
+unsigned int virt_get_caches(const VirtMachineState *vms, CPUCoreCaches *caches)
321
+{
322
+ int num_cache = 0;
323
+ ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(0)); /* assume homogeneous CPUs */
324
+ ARMISARegisters *isar = &armcpu->isar;
325
+ uint32_t clidr = GET_IDREG(isar, CLIDR);
326
+
327
+ for (int cache_level = 1; cache_level <= CLIDR_CTYPE_MAX_CACHE_LEVEL;
328
+ cache_level++) {
329
+ uint8_t ctype =
330
+ (clidr >> (3 * (cache_level - 1))) & CLIDR_CTYPE_MAX_CACHE_LEVEL;
331
+
332
+ if (ctype == CLIDR_CTYPE_NO_CACHE) {
333
+ /*
334
+ * If a "No cache" cache type is found it means no manageable caches
335
+ * exist at further-out levels of hierarchy, so ignore them.
336
+ */
337
+ break;
338
+ } else if (ctype == CLIDR_CTYPE_SEPARATE_I_D_CACHES) {
339
+ /*
340
+ * Create separate D and I caches. D-cache is stored first.
341
+ */
342
+ enum CacheType cache_type;
343
+ for (cache_type = DATA_CACHE; cache_type <= INSTRUCTION_CACHE;
344
+ cache_type++) {
345
+ set_cpu_cache(&caches[num_cache++], cache_type, cache_level,
346
+ cache_type == INSTRUCTION_CACHE ? true : false);
347
+ }
348
+ } else if (ctype == CLIDR_CTYPE_UNIFIED_CACHE) {
349
+ set_cpu_cache(&caches[num_cache++], UNIFIED_CACHE, cache_level,
350
+ false);
351
+ } else if (ctype == CLIDR_CTYPE_D_CACHE) {
352
+ set_cpu_cache(&caches[num_cache++], DATA_CACHE, cache_level, false);
353
+ } else if (ctype == CLIDR_CTYPE_I_CACHE) {
354
+ set_cpu_cache(&caches[num_cache++], INSTRUCTION_CACHE, cache_level,
355
+ true);
356
+ } else {
357
+ error_setg(&error_abort, "Unrecognized cache type");
358
+ return 0;
359
+ }
360
+ }
361
+
362
+ return num_cache;
363
+}
364
+
365
static void create_fdt(VirtMachineState *vms)
366
{
367
MachineState *ms = MACHINE(vms);
@@ -431,13 +512,124 @@ static void fdt_add_timer_nodes(const VirtMachineState *vms)
512
}
513
}
514
515
+static void add_cache_node(void *fdt, char *nodepath, CPUCoreCaches cache,
516
+ uint32_t *next_level)
517
+{
518
+ /* Assume L2/3 are unified caches. */
519
+
520
+ uint32_t phandle;
521
+
522
+ qemu_fdt_add_path(fdt, nodepath);
523
+ phandle = qemu_fdt_alloc_phandle(fdt);
524
+ qemu_fdt_setprop_cell(fdt, nodepath, "phandle", phandle);
525
+ qemu_fdt_setprop_cell(fdt, nodepath, "cache-level", cache.level);
526
+ qemu_fdt_setprop_cell(fdt, nodepath, "cache-size", cache.size);
527
+ qemu_fdt_setprop_cell(fdt, nodepath, "cache-block-size", cache.linesize);
528
+ qemu_fdt_setprop_cell(fdt, nodepath, "cache-sets", cache.sets);
529
+ qemu_fdt_setprop(fdt, nodepath, "cache-unified", NULL, 0);
530
+ qemu_fdt_setprop_string(fdt, nodepath, "compatible", "cache");
531
+ if (cache.level != 3) {
532
+ /* top level cache doesn't have next-level-cache property */
533
+ qemu_fdt_setprop_cell(fdt, nodepath, "next-level-cache", *next_level);
534
+ }
535
+
536
+ *next_level = phandle;
537
+}
538
+
539
+static bool add_cpu_cache_hierarchy(void *fdt, CPUCoreCaches* cache,
540
+ uint32_t cache_cnt,
541
+ uint32_t top_level,
542
+ uint32_t bottom_level,
543
+ uint32_t cpu_id,
544
+ uint32_t *next_level) {
545
+ bool found_cache = false;
546
+
547
+ for (int level = top_level; level >= bottom_level; level--) {
548
+ for (int i = 0; i < cache_cnt; i++) {
549
+ char *nodepath;
550
+
551
+ if (i != level) {
552
+ continue;
553
+ }
554
+
555
+ nodepath = g_strdup_printf("/cpus/cpu@%d/l%d-cache",
556
+ cpu_id, level);
557
+ add_cache_node(fdt, nodepath, cache[i], next_level);
558
+ found_cache = true;
559
+ g_free(nodepath);
560
+
561
+ }
562
+ }
563
+
564
+ return found_cache;
565
+}
566
+
567
+static void set_cache_properties(void *fdt, const char *nodename,
568
+ const char *prefix, CPUCoreCaches cache)
569
+{
570
+ char prop_name[64];
571
+
572
+ snprintf(prop_name, sizeof(prop_name), "%s-block-size", prefix);
573
+ qemu_fdt_setprop_cell(fdt, nodename, prop_name, cache.linesize);
574
+
575
+ snprintf(prop_name, sizeof(prop_name), "%s-size", prefix);
576
+ qemu_fdt_setprop_cell(fdt, nodename, prop_name, cache.size);
577
+
578
+ snprintf(prop_name, sizeof(prop_name), "%s-sets", prefix);
579
+ qemu_fdt_setprop_cell(fdt, nodename, prop_name, cache.sets);
580
+}
581
+
582
+static bool partial_cache_description(const MachineState *ms, int num_caches)
583
+{
584
+ assert(num_caches - 1 < CACHE_LEVEL_AND_TYPE__MAX);
585
+ enum CpuTopologyLevel topo_level;
586
+ enum CacheLevelAndType cache_level;
587
+
588
+ for (cache_level = 0; cache_level < num_caches; cache_level++) {
589
+ topo_level = machine_get_cache_topo_level(ms, cache_level);
590
+ if (topo_level == CPU_TOPOLOGY_LEVEL_DEFAULT) {
591
+ /* No topology level described for this cache level. */
592
+ return true;
593
+ }
594
+ }
595
+
596
+ return false;
597
+}
598
+
599
static void fdt_add_cpu_nodes(const VirtMachineState *vms)
600
{
601
int cpu;
602
int addr_cells = 1;
603
const MachineState *ms = MACHINE(vms);
604
+ const MachineClass *mc = MACHINE_GET_CLASS(ms);
605
const VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms);
606
int smp_cpus = ms->smp.cpus;
607
+ int socket_id, cluster_id, core_id;
608
+ uint32_t next_level = 0;
609
+ uint32_t socket_offset = 0;
610
+ uint32_t cluster_offset = 0;
611
+ uint32_t core_offset = 0;
612
+ int last_socket = -1;
613
+ int last_cluster = -1;
614
+ int last_core = -1;
615
+ int top_node = 3;
616
+ int top_cluster = 3;
617
+ int top_core = 3;
618
+ int bottom_node = 3;
619
+ int bottom_cluster = 3;
620
+ int bottom_core = 3;
621
+ unsigned int num_cache;
622
+ CPUCoreCaches caches[CPU_MAX_CACHES];
623
+ bool cache_created = false;
624
+ bool cache_at_topo_level;
625
+
626
+ num_cache = virt_get_caches(vms, caches);
627
+
628
+ if (mc->smp_props.has_caches &&
629
+ partial_cache_description(ms, num_cache)) {
630
+ error_setg(&error_fatal, "Missing cache description");
631
+ return;
632
+ }
633
634
/*
635
* See Linux Documentation/devicetree/bindings/arm/cpus.yaml
@@ -466,9 +658,14 @@ static void fdt_add_cpu_nodes(const VirtMachineState *vms)
658
qemu_fdt_setprop_cell(ms->fdt, "/cpus", "#size-cells", 0x0);
659
660
for (cpu = smp_cpus - 1; cpu >= 0; cpu--) {
661
+ socket_id = cpu / (ms->smp.clusters * ms->smp.cores * ms->smp.threads);
662
+ cluster_id = cpu / (ms->smp.cores * ms->smp.threads) % ms->smp.clusters;
663
+ core_id = cpu / ms->smp.threads % ms->smp.cores;
664
+
665
char *nodename = g_strdup_printf("/cpus/cpu@%d", cpu);
666
ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(cpu));
667
CPUState *cs = CPU(armcpu);
668
+ const char *prefix = NULL;
669
670
qemu_fdt_add_subnode(ms->fdt, nodename);
671
qemu_fdt_setprop_string(ms->fdt, nodename, "device_type", "cpu");
@@ -498,6 +695,136 @@ static void fdt_add_cpu_nodes(const VirtMachineState *vms)
695
qemu_fdt_alloc_phandle(ms->fdt));
696
}
697
698
+ if (!vmc->no_cpu_topology && num_cache) {
699
+ for (uint8_t i = 0; i < num_cache; i++) {
700
+ /* Only level 1 in the CPU entry. */
701
+ if (caches[i].level > 1) {
702
+ continue;
703
+ }
704
+
705
+ if (caches[i].type == INSTRUCTION_CACHE) {
706
+ prefix = "i-cache";
707
+ } else if (caches[i].type == DATA_CACHE) {
708
+ prefix = "d-cache";
709
+ } else if (caches[i].type == UNIFIED_CACHE) {
710
+ error_setg(&error_fatal,
711
+ "Unified type is not implemented at level %d",
712
+ caches[i].level);
713
+ return;
714
+ } else {
715
+ error_setg(&error_fatal, "Undefined cache type");
716
+ return;
717
+ }
718
+
719
+ set_cache_properties(ms->fdt, nodename, prefix, caches[i]);
720
+ }
721
+ }
722
+
723
+ if (socket_id != last_socket) {
724
+ bottom_node = top_node;
725
+ /* This assumes socket as the highest topological level. */
726
+ socket_offset = 0;
727
+ cluster_offset = 0;
728
+ cache_at_topo_level =
729
+ machine_find_lowest_level_cache_at_topo_level(ms,
730
+ &bottom_node,
731
+ CPU_TOPOLOGY_LEVEL_SOCKET);
732
+ if (cache_at_topo_level) {
733
+ if (bottom_node == 1 && !virt_is_acpi_enabled(vms))
734
+ error_setg(
735
+ &error_fatal,
736
+ "Cannot share L1 at socket_id %d."
737
+ "DT limitation on sharing at cache level = 1",
738
+ socket_id);
739
+
740
+ cache_created = add_cpu_cache_hierarchy(ms->fdt, caches,
741
+ num_cache,
742
+ top_node,
743
+ bottom_node, cpu,
744
+ &socket_offset);
745
+
746
+ if (!cache_created) {
747
+ error_setg(&error_fatal,
748
+ "Socket: No caches at levels %d-%d",
749
+ top_node, bottom_node);
750
+ return;
751
+ }
752
+
753
+ top_cluster = bottom_node - 1;
754
+ }
755
+
756
+ last_socket = socket_id;
757
+ }
758
+
759
+ if (cluster_id != last_cluster) {
760
+ bottom_cluster = top_cluster;
761
+ cluster_offset = socket_offset;
762
+ core_offset = 0;
763
+ cache_at_topo_level =
764
+ machine_find_lowest_level_cache_at_topo_level(ms,
765
+ &bottom_cluster,
766
+ CPU_TOPOLOGY_LEVEL_CLUSTER);
767
+ if (cache_at_topo_level) {
768
+ cache_created = add_cpu_cache_hierarchy(ms->fdt, caches,
769
+ num_cache,
770
+ top_cluster,
771
+ bottom_cluster, cpu,
772
+ &cluster_offset);
773
+ if (bottom_cluster == 1 && !virt_is_acpi_enabled(vms)) {
774
+ error_setg(&error_fatal,
775
+ "Cannot share L1 at socket_id %d, cluster_id %d. "
776
+ "DT limitation on sharing at cache level = 1.",
777
+ socket_id, cluster_id);
778
+ }
779
+
780
+ if (!cache_created) {
781
+ error_setg(&error_fatal,
782
+ "Cluster: No caches at levels %d-%d.",
783
+ top_cluster, bottom_cluster);
784
+ return;
785
+ }
786
+
787
+ top_core = bottom_cluster - 1;
788
+ } else if (top_cluster == bottom_node - 1) {
789
+ top_core = bottom_node - 1;
790
+ }
791
+
792
+ last_cluster = cluster_id;
793
+ }
794
+
795
+ if (core_id != last_core) {
796
+ bottom_core = top_core;
797
+ core_offset = cluster_offset;
798
+ cache_at_topo_level =
799
+ machine_find_lowest_level_cache_at_topo_level(ms,
800
+ &bottom_core,
801
+ CPU_TOPOLOGY_LEVEL_CORE);
802
+ if (cache_at_topo_level) {
803
+ if (bottom_core == 1 && top_core > 1) {
804
+ bottom_core++;
805
+ cache_created = add_cpu_cache_hierarchy(ms->fdt,
806
+ caches,
807
+ num_cache,
808
+ top_core,
809
+ bottom_core, cpu,
810
+ &core_offset);
811
+
812
+ if (!cache_created) {
813
+ error_setg(&error_fatal,
814
+ "Core: No caches at levels %d-%d",
815
+ top_core, bottom_core);
816
+ return;
817
+ }
818
+ }
819
+ }
820
+
821
+ last_core = core_id;
822
+ }
823
+
824
+ next_level = core_offset;
825
+ qemu_fdt_setprop_cell(ms->fdt, nodename, "next-level-cache",
826
+ next_level);
827
+
828
g_free(nodename);
829
}
830
@@ -2912,7 +3239,7 @@ static void virt_set_oem_table_id(Object *obj, const char *value,
3239
}
3240
3241
2915
-bool virt_is_acpi_enabled(VirtMachineState *vms)
3242
+bool virt_is_acpi_enabled(const VirtMachineState *vms)
3243
{
3244
if (vms->acpi == ON_OFF_AUTO_OFF) {
3245
return false;
@@ -3496,6 +3823,12 @@ static void virt_machine_class_init(ObjectClass *oc, const void *data)
3823
hc->unplug = virt_machine_device_unplug_cb;
3824
mc->nvdimm_supported = true;
3825
mc->smp_props.clusters_supported = true;
3826
+
3827
+ /* Supported caches */
3828
+ mc->smp_props.cache_supported[CACHE_LEVEL_AND_TYPE_L1D] = true;
3829
+ mc->smp_props.cache_supported[CACHE_LEVEL_AND_TYPE_L1I] = true;
3830
+ mc->smp_props.cache_supported[CACHE_LEVEL_AND_TYPE_L2] = true;
3831
+ mc->smp_props.cache_supported[CACHE_LEVEL_AND_TYPE_L3] = true;
3832
mc->auto_enable_numa_with_memhp = true;
3833
mc->auto_enable_numa_with_memdev = true;
3834
/* platform instead of architectural choice */
include/hw/arm/virt.h
+16
-1
@@ -41,6 +41,7 @@
41
#include "system/kvm.h"
42
#include "hw/intc/arm_gicv3_common.h"
43
#include "qom/object.h"
44
+#include "hw/core/cpu.h"
45
46
#define NUM_GICV2M_SPIS 64
47
#define NUM_VIRTIO_TRANSPORTS 32
@@ -52,6 +53,8 @@
53
/* GPIO pins */
54
#define GPIO_PIN_POWER_BUTTON 3
55
56
+#define CPU_MAX_CACHES 16
57
+
58
enum {
59
VIRT_FLASH,
60
VIRT_MEM,
@@ -195,7 +198,19 @@ struct VirtMachineState {
198
OBJECT_DECLARE_TYPE(VirtMachineState, VirtMachineClass, VIRT_MACHINE)
199
200
void virt_acpi_setup(VirtMachineState *vms);
198
-bool virt_is_acpi_enabled(VirtMachineState *vms);
201
+bool virt_is_acpi_enabled(const VirtMachineState *vms);
202
+
203
+#define CLIDR_CTYPE_NO_CACHE 0x00
204
+#define CLIDR_CTYPE_I_CACHE 0x01
205
+#define CLIDR_CTYPE_D_CACHE 0x02
206
+#define CLIDR_CTYPE_SEPARATE_I_D_CACHES 0x03
207
+#define CLIDR_CTYPE_UNIFIED_CACHE 0x04
208
+#define CLIDR_CTYPE_MAX_CACHE_LEVEL 7
209
+
210
+unsigned int virt_get_caches(const VirtMachineState *vms,
211
+ CPUCoreCaches *caches);
212
+void set_cpu_cache(CPUCoreCaches *cpu_cache, enum CacheType cache_type,
213
+ int cache_level, bool is_i_cache);
214
215
/* Return number of redistributors that fit in the specified region */
216
static uint32_t virt_redist_capacity(VirtMachineState *vms, int region)
include/hw/core/cpu.h
+12
@@ -1218,4 +1218,16 @@ enum CacheType {
1218
UNIFIED_CACHE
1219
};
1220
1221
+struct CPUCoreCaches {
1222
+ enum CacheType type;
1223
+ uint32_t sets;
1224
+ uint32_t size;
1225
+ uint32_t level;
1226
+ uint16_t linesize;
1227
+ uint8_t attributes; /* write policy: 0x0 write back, 0x1 write through */
1228
+ uint8_t associativity;
1229
+};
1230
+
1231
+typedef struct CPUCoreCaches CPUCoreCaches;
1232
+
1233
#endif