The RISC-V architecture specification says that memory read/writes are

not ordered with respect to mmio read/writes.  This appears to happen on
T-Head C920 cores as I'm seeing interrupts being enabled before the lock
is released in mtx_leave() despite program order releasing the lock
before enabling interrupts.  This is fixed by adding the necessary fences
in more or less the same places where Linux uses them.

ok patrick@, jca@

diff --git a/sys/arch/riscv64/include/bus.h b/sys/arch/riscv64/include/bus.h
index 14a324c..9e64499 100644 (file)
--- a/sys/arch/riscv64/include/bus.h
+++ b/sys/arch/riscv64/include/bus.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: bus.h,v 1.5 2022/12/29 11:35:01 kettenis Exp $        */
+/*     $OpenBSD: bus.h,v 1.6 2024/03/27 23:10:18 kettenis Exp $        */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB Sweden.  All rights reserved.
@@ -265,8 +265,10 @@ bus_space_copy_1(void *v, bus_space_handle_t h1, bus_size_t o1,
        char *s = (char *)(h1 + o1);
        char *d = (char *)(h2 + o2);
 
+       __asm volatile ("fence w,o" ::: "memory");
        while (c--)
                *d++ = *s++;
+       __asm volatile ("fence io,iwr" ::: "memory");
 }
 
 
@@ -277,8 +279,10 @@ bus_space_copy_2(void *v, bus_space_handle_t h1, bus_size_t o1,
        short *s = (short *)(h1 + o1);
        short *d = (short *)(h2 + o2);
 
+       __asm volatile ("fence w,o" ::: "memory");
        while (c--)
                *d++ = *s++;
+       __asm volatile ("fence io,iwr" ::: "memory");
 }
 
 static __inline void
@@ -288,8 +292,10 @@ bus_space_copy_4(void *v, bus_space_handle_t h1, bus_size_t o1,
        int *s = (int *)(h1 + o1);
        int *d = (int *)(h2 + o2);
 
+       __asm volatile ("fence w,o" ::: "memory");
        while (c--)
                *d++ = *s++;
+       __asm volatile ("fence io,iwr" ::: "memory");
 }
 
 static __inline void
@@ -299,8 +305,10 @@ bus_space_copy_8(void *v, bus_space_handle_t h1, bus_size_t o1,
        int64_t *s = (int64_t *)(h1 + o1);
        int64_t *d = (int64_t *)(h2 + o2);
 
+       __asm volatile ("fence w,o" ::: "memory");
        while (c--)
                *d++ = *s++;
+       __asm volatile ("fence io,iwr" ::: "memory");
 }
 
 /*----------------------------------------------------------------------------*/

diff --git a/sys/arch/riscv64/riscv64/bus_space.c b/sys/arch/riscv64/riscv64/bus_space.c
index d20ffb1..4884d68 100644 (file)
--- a/sys/arch/riscv64/riscv64/bus_space.c
+++ b/sys/arch/riscv64/riscv64/bus_space.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: bus_space.c,v 1.2 2021/05/12 01:20:52 jsg Exp $       */
+/*     $OpenBSD: bus_space.c,v 1.3 2024/03/27 23:10:18 kettenis Exp $  */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -64,53 +64,69 @@ bus_space_t *fdt_cons_bs_tag = &riscv64_bs_tag;
 uint8_t
 generic_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-       return *(volatile uint8_t *)(h + o);
+       uint8_t val = *(volatile uint8_t *)(h + o);
+       __asm volatile ("fence i,ir" ::: "memory");
+       return val;
 }
 
 uint16_t
 generic_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-       return *(volatile uint16_t *)(h + o);
+       uint16_t val = *(volatile uint16_t *)(h + o);
+       __asm volatile ("fence i,ir" ::: "memory");
+       return val;
 }
 
 uint32_t
 generic_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-       return *(volatile uint32_t *)(h + o);
+       uint32_t val = *(volatile uint32_t *)(h + o);
+       __asm volatile ("fence i,ir" ::: "memory");
+       return val;
 }
 
 uint64_t
 generic_space_read_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-       return *(volatile uint64_t *)(h + o);
+       uint64_t val = *(volatile uint64_t *)(h + o);
+       __asm volatile ("fence i,ir" ::: "memory");
+       return val;
 }
 
 void
 generic_space_write_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint8_t v)
 {
+       __asm volatile ("fence w,o" ::: "memory");
        *(volatile uint8_t *)(h + o) = v;
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint16_t v)
 {
+       __asm volatile ("fence w,o" ::: "memory");
        *(volatile uint16_t *)(h + o) = v;
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint32_t v)
 {
+       __asm volatile ("fence w,o" ::: "memory");
        *(volatile uint32_t *)(h + o) = v;
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint64_t v)
 {
+       __asm volatile ("fence w,o" ::: "memory");
        *(volatile uint64_t *)(h + o) = v;
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -123,6 +139,7 @@ generic_space_read_raw_2(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
                *(uint16_t *)buf = *addr;
                buf += 2;
        }
+       __asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -130,11 +147,13 @@ generic_space_write_raw_2(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
        volatile uint16_t *addr = (volatile uint16_t *)(h + o);
+       __asm volatile ("fence w,o" ::: "memory");
        len >>= 1;
        while (len-- != 0) {
                *addr = *(uint16_t *)buf;
                buf += 2;
        }
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -147,6 +166,7 @@ generic_space_read_raw_4(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
                *(uint32_t *)buf = *addr;
                buf += 4;
        }
+       __asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -154,11 +174,13 @@ generic_space_write_raw_4(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
        volatile uint32_t *addr = (volatile uint32_t *)(h + o);
+       __asm volatile ("fence w,o" ::: "memory");
        len >>= 2;
        while (len-- != 0) {
                *addr = *(uint32_t *)buf;
                buf += 4;
        }
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -171,6 +193,7 @@ generic_space_read_raw_8(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
                *(uint64_t *)buf = *addr;
                buf += 8;
        }
+       __asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -178,11 +201,13 @@ generic_space_write_raw_8(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
        volatile uint64_t *addr = (volatile uint64_t *)(h + o);
+       __asm volatile ("fence w,o" ::: "memory");
        len >>= 3;
        while (len-- != 0) {
                *addr = *(uint64_t *)buf;
                buf += 8;
        }
+       __asm volatile ("fence o,w" ::: "memory");
 }
 
 int