x86/speculation: Prepare entry code for Spectre v1 swapgs mitigations [Linux 4.14.137]

This Linux kernel change "x86/speculation: Prepare entry code for Spectre v1 swapgs mitigations" is included in the Linux 4.14.137 release. This change is authored by Josh Poimboeuf <jpoimboe [at] redhat.com> on Mon Jul 8 11:52:25 2019 -0500. The commit for this change in Linux stable tree is 83e6e48 (patch) which is from upstream commit 18ec54f. The same Linux upstream change may have been applied to various maintained Linux releases and you can find all Linux releases containing changes from upstream 18ec54f.

x86/speculation: Prepare entry code for Spectre v1 swapgs mitigations

commit 18ec54fdd6d18d92025af097cd042a75cf0ea24c upstream

Spectre v1 isn't only about array bounds checks.  It can affect any
conditional checks.  The kernel entry code interrupt, exception, and NMI
handlers all have conditional swapgs checks.  Those may be problematic in
the context of Spectre v1, as kernel code can speculatively run with a user
GS.

For example:

    if (coming from user space)
        swapgs
    mov %gs:<percpu_offset>, %reg
    mov (%reg), %reg1

When coming from user space, the CPU can speculatively skip the swapgs, and
then do a speculative percpu load using the user GS value.  So the user can
speculatively force a read of any kernel value.  If a gadget exists which
uses the percpu value as an address in another load/store, then the
contents of the kernel value may become visible via an L1 side channel
attack.

A similar attack exists when coming from kernel space.  The CPU can
speculatively do the swapgs, causing the user GS to get used for the rest
of the speculative window.

The mitigation is similar to a traditional Spectre v1 mitigation, except:

  a) index masking isn't possible; because the index (percpu offset)
     isn't user-controlled; and

  b) an lfence is needed in both the "from user" swapgs path and the
     "from kernel" non-swapgs path (because of the two attacks described
     above).

The user entry swapgs paths already have SWITCH_TO_KERNEL_CR3, which has a
CR3 write when PTI is enabled.  Since CR3 writes are serializing, the
lfences can be skipped in those cases.

On the other hand, the kernel entry swapgs paths don't depend on PTI.

To avoid unnecessary lfences for the user entry case, create two separate
features for alternative patching:

  X86_FEATURE_FENCE_SWAPGS_USER
  X86_FEATURE_FENCE_SWAPGS_KERNEL

Use these features in entry code to patch in lfences where needed.

The features aren't enabled yet, so there's no functional change.

Signed-off-by: Josh Poimboeuf <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Reviewed-by: Dave Hansen <[email protected]>
Signed-off-by: Greg Kroah-Hartman <[email protected]>

There are 38 lines of Linux source code added/deleted in this change. Code changes to Linux kernel are as follows.

 arch/x86/entry/calling.h           | 17 +++++++++++++++++
 arch/x86/entry/entry_64.S          | 19 +++++++++++++++++--
 arch/x86/include/asm/cpufeatures.h |  2 ++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 5d10b7a..557c1bd 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -332,6 +332,23 @@

 #endif

+/*
+ * Mitigate Spectre v1 for conditional swapgs code paths.
+ *
+ * FENCE_SWAPGS_USER_ENTRY is used in the user entry swapgs code path, to
+ * prevent a speculative swapgs when coming from kernel space.
+ *
+ * FENCE_SWAPGS_KERNEL_ENTRY is used in the kernel entry non-swapgs code path,
+ * to prevent the swapgs from getting speculatively skipped when coming from
+ * user space.
+ */
+.macro FENCE_SWAPGS_USER_ENTRY
+   ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_USER
+.endm
+.macro FENCE_SWAPGS_KERNEL_ENTRY
+   ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL
+.endm
+
 #endif /* CONFIG_X86_64 */

 /*
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index b2524d3..5ec66fa 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -531,9 +531,12 @@ END(irq_entries_start)
    testb   $3, CS-ORIG_RAX(%rsp)
    jz  1f
    SWAPGS
+   FENCE_SWAPGS_USER_ENTRY
    call    switch_to_thread_stack
+   jmp 2f
 1:
-
+   FENCE_SWAPGS_KERNEL_ENTRY
+2:
    PUSH_AND_CLEAR_REGS
    ENCODE_FRAME_POINTER

@@ -1146,6 +1149,12 @@ ENTRY(paranoid_entry)

 1:
    SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
+   /*
+    * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
+    * unconditional CR3 write, even in the PTI case.  So do an lfence
+    * to prevent GS speculation, regardless of whether PTI is enabled.
+    */
+   FENCE_SWAPGS_KERNEL_ENTRY

    ret
 END(paranoid_entry)
@@ -1195,6 +1204,7 @@ ENTRY(error_entry)
     * from user mode due to an IRET fault.
     */
    SWAPGS
+   FENCE_SWAPGS_USER_ENTRY
    /* We have user CR3.  Change to kernel CR3. */
    SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

@@ -1216,6 +1226,8 @@ ENTRY(error_entry)
    CALL_enter_from_user_mode
    ret

+.Lerror_entry_done_lfence:
+   FENCE_SWAPGS_KERNEL_ENTRY
 .Lerror_entry_done:
    TRACE_IRQS_OFF
    ret
@@ -1234,7 +1246,7 @@ ENTRY(error_entry)
    cmpq    %rax, RIP+8(%rsp)
    je  .Lbstep_iret
    cmpq    $.Lgs_change, RIP+8(%rsp)
-   jne .Lerror_entry_done
+   jne .Lerror_entry_done_lfence

    /*
     * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
@@ -1242,6 +1254,7 @@ ENTRY(error_entry)
     * .Lgs_change's error handler with kernel gsbase.
     */
    SWAPGS
+   FENCE_SWAPGS_USER_ENTRY
    SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
    jmp .Lerror_entry_done

@@ -1256,6 +1269,7 @@ ENTRY(error_entry)
     * gsbase and CR3.  Switch to kernel gsbase and CR3:
     */
    SWAPGS
+   FENCE_SWAPGS_USER_ENTRY
    SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

    /*
@@ -1347,6 +1361,7 @@ ENTRY(nmi)

    swapgs
    cld
+   FENCE_SWAPGS_USER_ENTRY
    SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
    movq    %rsp, %rdx
    movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 315e87a..cf79c3f 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -281,6 +281,8 @@
 #define X86_FEATURE_CQM_OCCUP_LLC  (11*32+ 1) /* LLC occupancy monitoring */
 #define X86_FEATURE_CQM_MBM_TOTAL  (11*32+ 2) /* LLC Total MBM monitoring */
 #define X86_FEATURE_CQM_MBM_LOCAL  (11*32+ 3) /* LLC Local MBM monitoring */
+#define X86_FEATURE_FENCE_SWAPGS_USER  (11*32+ 4) /* "" LFENCE in user entry SWAPGS path */
+#define X86_FEATURE_FENCE_SWAPGS_KERNEL    (11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */

 /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */
 #define X86_FEATURE_CLZERO     (13*32+ 0) /* CLZERO instruction */

Leave a Reply

Your email address will not be published. Required fields are marked *