Direct access to physical address is a bad idea, but sometimes, for debug we need it. The trick used below is to disable MMU for only one instruction that do the read with fully configured environment and then re enabling MMU. Read and enable instructions should be in the pipe. The code works for arm11 but should also works for arm9 and maybe CortexA 32 bits series. Primitives has been found in FreeBSD 10 source code (sys/arm/arm/).

read_arm_phys_addr() must be called in supervisor mode (kernel mode).

#include <stdint.h>

/* CPU control register (CP15 register 1) */
#define CPU_CONTROL_MMU_ENABLE  0x00000001 /* M: MMU/Protection unit enable */
#define CPU_CONTROL_DC_ENABLE   0x00000004 /* C: IDC/DC enable */
#define CPU_CONTROL_WBUF_ENABLE 0x00000008 /* W: Write buffer enable */
#define CPU_CONTROL_BPRD_ENABLE 0x00000800 /* Z: Branch prediction enable */
#define CPU_CONTROL_IC_ENABLE   0x00001000 /* I: IC enable */

uint32_t read_arm_phys_addr(void* addr)
{
    uint32_t value;

    __asm __volatile (
        // Load address                                                                                                                
        "mov     r0, %1\n\r"
        // Disable IRQ, FIQ and abort                                                                                                  
        "cpsid   ifa\n\r"
        // Disable MMU and cache                                                                                                       
        "mrc     p15, 0, r1, c1, c0, 0\n\r"
        "bic     r2, r1, %2\n\r"
        "bic     r2, r2, %3\n\r"
        "bic     r2, r2, %4\n\r"
        "mcr     p15, 0, r2, c1, c0, 0\n\r"
        "nop\n\r"
        "nop\n\r"
        "nop\n\r"
        // Read data                                                                                                                   
        "ldr     r3, [r0]\n\r"
        // Enable MMU                                                                                                                  
        "mcr     p15, 0, r1, c1, c0, 0\n\r"
        "nop\n\r"
        "nop\n\r"
        "nop\n\r"
        // Enable IRQ, FIQ and abort                                                                                                   
        "cpsie   ifa\n\r"
        "mov     %0, r3\n\r"
        : "=r" (value)
        : "r" (addr), "i" (CPU_CONTROL_MMU_ENABLE | CPU_CONTROL_DC_ENABLE | CPU_CONTROL_WBUF_ENABLE),
        "i" (CPU_CONTROL_IC_ENABLE), "i" (CPU_CONTROL_BPRD_ENABLE)
        : "r0", "r1", "r2", "r3");

    return value;
}