sparc: Improve VDSO instruction patching.
authorDavid S. Miller <davem@davemloft.net>
Mon, 22 Oct 2018 04:44:33 +0000 (21:44 -0700)
committerDavid S. Miller <davem@davemloft.net>
Mon, 22 Oct 2018 22:22:14 +0000 (15:22 -0700)
The current VDSO patch mechanism has several problems:

1) It assumes how gcc will emit a function, with a register
   window, an initial save instruction and then immediately
   the %tick read when compiling vread_tick().

   There is no such guarantees, code generation could change
   at any time, gcc could put a nop between the save and
   the %tick read, etc.

   So this is extremely fragile and would fail some day.

2) It disallows us to properly inline vread_tick() into the callers
   and thus get the best possible code sequences.

So fix this to patch properly, with location based annotations.

We have to be careful because we cannot do it the way we do
patches elsewhere in the kernel.  Those use a sequence like:

1:
insn
.section .whatever_patch, "ax"
.word 1b
replacement_insn
.previous

This is a dynamic shared object, so that .word cannot be resolved at
build time, and thus cannot be used to execute the patches when the
kernel initializes the images.

Even trying to use label difference equations doesn't work in the
above kind of scheme:

1:
insn
.section .whatever_patch, "ax"
.word . - 1b
replacement_insn
.previous

The assembler complains that it cannot resolve that computation.
The issue is that this is contained in an executable section.

Borrow the sequence used by x86 alternatives, which is:

1:
insn
.pushsection .whatever_patch, "a"
.word . - 1b, . - 1f
.popsection
.pushsection .whatever_patch_replacements, "ax"
1:
replacement_insn
.previous

This works, allows us to inline vread_tick() as much as we like, and
can be used for arbitrary kinds of VDSO patching in the future.

Also, reverse the condition for patching.  Most systems are %stick
based, so if we only patch on %tick systems the patching code will
get little or no testing.

Signed-off-by: David S. Miller <davem@davemloft.net>
arch/sparc/include/asm/vdso.h
arch/sparc/kernel/time_64.c
arch/sparc/vdso/vclock_gettime.c
arch/sparc/vdso/vdso-layout.lds.S
arch/sparc/vdso/vdso2c.c
arch/sparc/vdso/vdso2c.h
arch/sparc/vdso/vma.c

index 93b6287..56836eb 100644 (file)
@@ -8,10 +8,10 @@
 struct vdso_image {
        void *data;
        unsigned long size;   /* Always a multiple of PAGE_SIZE */
+
+       unsigned long tick_patch, tick_patch_len;
+
        long sym_vvar_start;  /* Negative offset to the vvar area */
-       long sym_vread_tick; /* Start of vread_tick section */
-       long sym_vread_tick_patch_start; /* Start of tick read */
-       long sym_vread_tick_patch_end;   /* End of tick read */
 };
 
 #ifdef CONFIG_SPARC64
index f0eba72..5f356dc 100644 (file)
@@ -53,8 +53,6 @@
 
 DEFINE_SPINLOCK(rtc_lock);
 
-unsigned int __read_mostly vdso_fix_stick;
-
 #ifdef CONFIG_SMP
 unsigned long profile_pc(struct pt_regs *regs)
 {
@@ -838,7 +836,6 @@ void __init time_init_early(void)
                } else {
                        init_tick_ops(&tick_operations);
                        clocksource_tick.archdata.vclock_mode = VCLOCK_TICK;
-                       vdso_fix_stick = 1;
                }
        } else {
                init_tick_ops(&stick_operations);
index 75dca9a..a0c8a4b 100644 (file)
@@ -105,29 +105,36 @@ static notrace noinline u64
 vread_tick(void) {
        u64     ret;
 
-       __asm__ __volatile__("rd        %%asr24, %0 \n"
-                            ".section  .vread_tick_patch, \"ax\" \n"
-                            "rd        %%tick, %0 \n"
-                            ".previous \n"
-                            : "=&r" (ret));
+       __asm__ __volatile__("1:\n\t"
+                            "rd                %%tick, %0\n\t"
+                            ".pushsection      .tick_patch, \"a\"\n\t"
+                            ".word             1b - ., 1f - .\n\t"
+                            ".popsection\n\t"
+                            ".pushsection      .tick_patch_replacement, \"ax\"\n\t"
+                            "1:\n\t"
+                            "rd                %%asr24, %0\n\t"
+                            ".popsection\n"
+                            : "=r" (ret));
        return ret & ~TICK_PRIV_BIT;
 }
 #else
 static notrace noinline u64
 vread_tick(void)
 {
-       unsigned int lo, hi;
-
-       __asm__ __volatile__("rd        %%asr24, %%g1\n\t"
-                            "srlx      %%g1, 32, %1\n\t"
-                            "srl       %%g1, 0, %0\n"
-                            ".section  .vread_tick_patch, \"ax\" \n"
-                            "rd        %%tick, %%g1\n"
-                            ".previous \n"
-                            : "=&r" (lo), "=&r" (hi)
-                            :
-                            : "g1");
-       return lo | ((u64)hi << 32);
+       register unsigned long long ret asm("o4");
+
+       __asm__ __volatile__("1:\n\t"
+                            "rd                %%tick, %L0\n\t"
+                            "srlx              %L0, 32, %H0\n\t"
+                            ".pushsection      .tick_patch, \"a\"\n\t"
+                            ".word             1b - ., 1f - .\n\t"
+                            ".popsection\n\t"
+                            ".pushsection      .tick_patch_replacement, \"ax\"\n\t"
+                            "1:\n\t"
+                            "rd                %%asr24, %L0\n\t"
+                            ".popsection\n"
+                            : "=r" (ret));
+       return ret;
 }
 #endif
 
index f2c83ab..ed36d49 100644 (file)
@@ -73,11 +73,8 @@ SECTIONS
 
        .text           : { *(.text*) }                 :text   =0x90909090,
 
-       .vread_tick_patch : {
-               vread_tick_patch_start = .;
-               *(.vread_tick_patch)
-               vread_tick_patch_end = .;
-       }
+       .tick_patch       : { *(.tick_patch) }          :text
+       .tick_patch_insns : { *(.tick_patch_insns) }    :text
 
        /DISCARD/ : {
                *(.discard)
index 9f5b1cd..ab75041 100644 (file)
@@ -63,9 +63,6 @@ enum {
        sym_vvar_start,
        sym_VDSO_FAKE_SECTION_TABLE_START,
        sym_VDSO_FAKE_SECTION_TABLE_END,
-       sym_vread_tick,
-       sym_vread_tick_patch_start,
-       sym_vread_tick_patch_end
 };
 
 struct vdso_sym {
@@ -81,9 +78,6 @@ struct vdso_sym required_syms[] = {
        [sym_VDSO_FAKE_SECTION_TABLE_END] = {
                "VDSO_FAKE_SECTION_TABLE_END", 0
        },
-       [sym_vread_tick] = {"vread_tick", 1},
-       [sym_vread_tick_patch_start] = {"vread_tick_patch_start", 1},
-       [sym_vread_tick_patch_end] = {"vread_tick_patch_end", 1}
 };
 
 __attribute__((format(printf, 1, 2))) __attribute__((noreturn))
index 808decb..4df005c 100644 (file)
@@ -17,10 +17,11 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
        unsigned long mapping_size;
        int i;
        unsigned long j;
-
-       ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr;
+       ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr, *secstrings_hdr,
+               *patch_sec = NULL;
        ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr;
        ELF(Dyn) *dyn = 0, *dyn_end = 0;
+       const char *secstrings;
        INT_BITS syms[NSYMS] = {};
 
        ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff));
@@ -63,11 +64,18 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
        }
 
        /* Walk the section table */
+       secstrings_hdr = raw_addr + GET_BE(&hdr->e_shoff) +
+               GET_BE(&hdr->e_shentsize)*GET_BE(&hdr->e_shstrndx);
+       secstrings = raw_addr + GET_BE(&secstrings_hdr->sh_offset);
        for (i = 0; i < GET_BE(&hdr->e_shnum); i++) {
                ELF(Shdr) *sh = raw_addr + GET_BE(&hdr->e_shoff) +
                        GET_BE(&hdr->e_shentsize) * i;
                if (GET_BE(&sh->sh_type) == SHT_SYMTAB)
                        symtab_hdr = sh;
+
+               if (!strcmp(secstrings + GET_BE(&sh->sh_name),
+                           ".tick_patch"))
+                       patch_sec = sh;
        }
 
        if (!symtab_hdr)
@@ -134,6 +142,12 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
        fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name);
        fprintf(outfile, "\t.data = raw_data,\n");
        fprintf(outfile, "\t.size = %lu,\n", mapping_size);
+       if (patch_sec) {
+               fprintf(outfile, "\t.tick_patch = %lu,\n",
+                       (unsigned long)GET_BE(&patch_sec->sh_offset));
+               fprintf(outfile, "\t.tick_patch_len = %lu,\n",
+                       (unsigned long)GET_BE(&patch_sec->sh_size));
+       }
        for (i = 0; i < NSYMS; i++) {
                if (required_syms[i].export && syms[i])
                        fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n",
index 5eaff3c..8874a27 100644 (file)
@@ -16,6 +16,8 @@
 #include <linux/linkage.h>
 #include <linux/random.h>
 #include <linux/elf.h>
+#include <asm/cacheflush.h>
+#include <asm/spitfire.h>
 #include <asm/vdso.h>
 #include <asm/vvar.h>
 #include <asm/page.h>
@@ -40,7 +42,25 @@ static struct vm_special_mapping vdso_mapping32 = {
 
 struct vvar_data *vvar_data;
 
-#define        SAVE_INSTR_SIZE 4
+struct tick_patch_entry {
+       s32 orig, repl;
+};
+
+static void stick_patch(const struct vdso_image *image)
+{
+       struct tick_patch_entry *p, *p_end;
+
+       p = image->data + image->tick_patch;
+       p_end = (void *)p + image->tick_patch_len;
+       while (p < p_end) {
+               u32 *instr = (void *)&p->orig + p->orig;
+               u32 *repl = (void *)&p->repl + p->repl;
+
+               *instr = *repl;
+               flushi(instr);
+               p++;
+       }
+}
 
 /*
  * Allocate pages for the vdso and vvar, and copy in the vdso text from the
@@ -68,21 +88,8 @@ int __init init_vdso_image(const struct vdso_image *image,
        if (!cpp)
                goto oom;
 
-       if (vdso_fix_stick) {
-               /*
-                * If the system uses %tick instead of %stick, patch the VDSO
-                * with instruction reading %tick instead of %stick.
-                */
-               unsigned int j, k = SAVE_INSTR_SIZE;
-               unsigned char *data = image->data;
-
-               for (j = image->sym_vread_tick_patch_start;
-                    j < image->sym_vread_tick_patch_end; j++) {
-
-                       data[image->sym_vread_tick + k] = data[j];
-                       k++;
-               }
-       }
+       if (tlb_type != spitfire)
+               stick_patch(image);
 
        for (i = 0; i < cnpages; i++) {
                cp = alloc_page(GFP_KERNEL);