26 Nov 2016

QEMU TCG backend

Author: derek0883@gmail.com

In previous post, I already know how TCG frontend module translate guest’s binary code into TCG intermediate code. I’m try to study how TCG backend module “compile” it to host’s native code.

Inside TCG backend

Firstly, I enabled out_asm log.

TCG frontend experiment

In order to get more information, I made the following changes:

diff --git a/disas.c b/disas.c
@@ -319,6 +319,9 @@ void disas(FILE *out, void *code, unsigned long size)
     for (pc = (uintptr_t)code; size > 0; pc += count, size -= count) {
         fprintf(out, "0x%08" PRIxPTR ":  ", pc);
         count = print_insn(pc, &s.info);
+               fprintf(out, "\t0x");
+               for (int i = 0; i < count; i++)
+                       fprintf(out, "%02x", ((unsigned char*)pc)[i]);
        fprintf(out, "\n");
$ ./ppc-softmmu/qemu-system-ppc -monitor stdio  images/cirros-0.3.4-powerpc-disk.img -trace events=/tmp/events -d in_asm,op,out_asm -D /tmp/qemu_ppc.log

Now in qemu_ppc.log, I found:

nip=fff00100 super=3 ir=0
translate opcode 48002420 (12 10 10 00) (big)
0xfff00100:  .long 0x48002420: b       0xfff02520

 ld_i32 tmp0,env,$0xfffffffffffffff8
 movi_i32 tmp1,$0x0
 brcond_i32 tmp0,tmp1,ne,$L0

 ---- fff00100
 movi_i32 nip,$0xfffffffffff02520
 exit_tb $0x0
 set_label $L0
 exit_tb $0x7ffff05ed013

OUT: [size=42]
0x7fffe9fff028:  mov    -0x8(%r14),%ebp 0x418b6ef8
0x7fffe9fff02c:  test   %ebp,%ebp   0x85ed
0x7fffe9fff02e:  jne    0x7fffe9fff046  0x0f8512000000
0x7fffe9fff034:  movl   $0xfff02520,0x26c(%r14) 0x41c7866c0200002025f0ff
0x7fffe9fff03f:  xor    %eax,%eax   0x33c0
0x7fffe9fff041:  jmpq   0x7fffe9fff016  0xe9d0ffffff
0x7fffe9fff046:  lea    0x65edfc6(%rip),%rax        # 0x7ffff05ed013    0x488d05c6df5e06
0x7fffe9fff04d:  jmpq   0x7fffe9fff016  0xe9c4ffffff

We already know At address 0xfff00100, PPC instruction “b 0xfff02520” was translated to TCG instruction “movi_i32 nip,$0xfffffffffff02520”. Now TCG backend translated to x86_64 instruction movl $0xfff02520,0x26c(%r14), then binary code is: 0x41c7866c0200002025f0ff. That because currently this piece of guest code has not been scheduled to host’s CPU yet, So it just save the PC 0xfff02520 to 0x26c(%r14). I believe It just like the context of thread scheduling in the kernel. To understand more about other instruction here, I guess I need dive deep into how CPU was emulated, that is another topic.

TCG backend flow

GDB back tracing dump:

#0  tcg_out_op (s=0x555556382400 <tcg_ctx>, opc=INDEX_op_ld_i32, args=0x7fffe93177f0, const_args=0x7fffe93177b0)
    at qemu-2.8.0-rc0/tcg/i386/tcg-target.inc.c:1798
#1  0x0000555555796d6e in tcg_reg_alloc_op (s=0x555556382400 ... opc=INDEX_op_ld_i32, ... >, 
    at qemu-2.8.0-rc0/tcg/tcg.c:2350
#2  0x000055555579796a in tcg_gen_code at qemu-2.8.0-rc0/tcg/tcg.c:2669
#3  0x000055555578a406 in tb_gen_code at qemu-2.8.0-rc0/translate-all.c:1339
#4  0x000055555578c405 in tb_find at qemu-2.8.0-rc0/cpu-exec.c:346
#5  0x000055555578cbff in cpu_exec at qemu-2.8.0-rc0/cpu-exec.c:637
#6  0x00005555557ce186 in tcg_cpu_exec at qemu-2.8.0-rc0/cpus.c:1117
#7  0x00005555557ce3ca in qemu_tcg_cpu_thread_fn at qemu-2.8.0-rc0/cpus.c:1197
#8  0x00007ffff65746fa in start_thread (arg=0x7fffe9318700) at pthread_create.c:333
#9  0x00007ffff62aab5d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

tcg_gen_code will decode each TCG instruction, then generate host code.

int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
    for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) {
        TCGOp * const op = &s->gen_op_buf[oi];
        TCGArg * const args = &s->gen_opparam_buf[op->args];
        TCGOpcode opc = op->opc;
        const TCGOpDef *def = &tcg_op_defs[opc];
        TCGLifeData arg_life = op->life;

        oi_next = op->next;

        switch (opc) {
        case INDEX_op_mov_i32:
        case INDEX_op_mov_i64:
            tcg_reg_alloc_mov(s, def, args, arg_life);


For movi_i32 nip,$0xfffffffffff02520, the opc is INDEX_op_mov_i32. I’m running QEMU on intel CPU, So tcg_reg_alloc_mov will call tcg_out_movi in tcg/i386/tcg-target.inc.c. x86_64 binary instruction will be stored in s->code_ptr.

static void tcg_out_movi(TCGContext *s, TCGType type,
                         TCGReg ret, tcg_target_long arg)
    tcg_target_long diff;

    if (arg == 0) {
        tgen_arithr(s, ARITH_XOR, ret, ret);
    if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
        tcg_out32(s, arg);
    if (arg == (int32_t)arg) {
        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
        tcg_out32(s, arg);

    /* Try a 7 byte pc-relative lea before the 10 byte movq.  */
    diff = arg - ((uintptr_t)s->code_ptr + 7);
    if (diff == (int32_t)diff) {
        tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
        tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
        tcg_out32(s, diff);

    tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
    tcg_out64(s, arg);