Merge tag 'iommu-updates-v4.17' of git://git.kernel.org/pub/scm/linux/kernel/git...
[muen/linux.git] / drivers / irqchip / irq-gic-v3-its.c
index b84bbcd..5416f2b 100644 (file)
@@ -33,6 +33,7 @@
 #include <linux/of_platform.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
+#include <linux/syscore_ops.h>
 
 #include <linux/irqchip.h>
 #include <linux/irqchip/arm-gic-v3.h>
@@ -46,6 +47,7 @@
 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING          (1ULL << 0)
 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375      (1ULL << 1)
 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144      (1ULL << 2)
+#define ITS_FLAGS_SAVE_SUSPEND_STATE           (1ULL << 3)
 
 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING    (1 << 0)
 
@@ -101,6 +103,8 @@ struct its_node {
        struct its_collection   *collections;
        struct fwnode_handle    *fwnode_handle;
        u64                     (*get_msi_base)(struct its_device *its_dev);
+       u64                     cbaser_save;
+       u32                     ctlr_save;
        struct list_head        its_device_list;
        u64                     flags;
        unsigned long           list_nr;
@@ -1875,16 +1879,6 @@ static void its_cpu_init_lpis(void)
                gic_data_rdist()->pend_page = pend_page;
        }
 
-       /* Disable LPIs */
-       val = readl_relaxed(rbase + GICR_CTLR);
-       val &= ~GICR_CTLR_ENABLE_LPIS;
-       writel_relaxed(val, rbase + GICR_CTLR);
-
-       /*
-        * Make sure any change to the table is observable by the GIC.
-        */
-       dsb(sy);
-
        /* set PROPBASE */
        val = (page_to_phys(gic_rdists->prop_page) |
               GICR_PROPBASER_InnerShareable |
@@ -1938,52 +1932,53 @@ static void its_cpu_init_lpis(void)
        dsb(sy);
 }
 
-static void its_cpu_init_collection(void)
+static void its_cpu_init_collection(struct its_node *its)
 {
-       struct its_node *its;
-       int cpu;
-
-       spin_lock(&its_lock);
-       cpu = smp_processor_id();
-
-       list_for_each_entry(its, &its_nodes, entry) {
-               u64 target;
+       int cpu = smp_processor_id();
+       u64 target;
 
-               /* avoid cross node collections and its mapping */
-               if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
-                       struct device_node *cpu_node;
+       /* avoid cross node collections and its mapping */
+       if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
+               struct device_node *cpu_node;
 
-                       cpu_node = of_get_cpu_node(cpu, NULL);
-                       if (its->numa_node != NUMA_NO_NODE &&
-                               its->numa_node != of_node_to_nid(cpu_node))
-                               continue;
-               }
+               cpu_node = of_get_cpu_node(cpu, NULL);
+               if (its->numa_node != NUMA_NO_NODE &&
+                       its->numa_node != of_node_to_nid(cpu_node))
+                       return;
+       }
 
+       /*
+        * We now have to bind each collection to its target
+        * redistributor.
+        */
+       if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
                /*
-                * We now have to bind each collection to its target
+                * This ITS wants the physical address of the
                 * redistributor.
                 */
-               if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
-                       /*
-                        * This ITS wants the physical address of the
-                        * redistributor.
-                        */
-                       target = gic_data_rdist()->phys_base;
-               } else {
-                       /*
-                        * This ITS wants a linear CPU number.
-                        */
-                       target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
-                       target = GICR_TYPER_CPU_NUMBER(target) << 16;
-               }
+               target = gic_data_rdist()->phys_base;
+       } else {
+               /* This ITS wants a linear CPU number. */
+               target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
+               target = GICR_TYPER_CPU_NUMBER(target) << 16;
+       }
 
-               /* Perform collection mapping */
-               its->collections[cpu].target_address = target;
-               its->collections[cpu].col_id = cpu;
+       /* Perform collection mapping */
+       its->collections[cpu].target_address = target;
+       its->collections[cpu].col_id = cpu;
 
-               its_send_mapc(its, &its->collections[cpu], 1);
-               its_send_invall(its, &its->collections[cpu]);
-       }
+       its_send_mapc(its, &its->collections[cpu], 1);
+       its_send_invall(its, &its->collections[cpu]);
+}
+
+static void its_cpu_init_collections(void)
+{
+       struct its_node *its;
+
+       spin_lock(&its_lock);
+
+       list_for_each_entry(its, &its_nodes, entry)
+               its_cpu_init_collection(its);
 
        spin_unlock(&its_lock);
 }
@@ -3041,6 +3036,113 @@ static void its_enable_quirks(struct its_node *its)
        gic_enable_quirks(iidr, its_quirks, its);
 }
 
+static int its_save_disable(void)
+{
+       struct its_node *its;
+       int err = 0;
+
+       spin_lock(&its_lock);
+       list_for_each_entry(its, &its_nodes, entry) {
+               void __iomem *base;
+
+               if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
+                       continue;
+
+               base = its->base;
+               its->ctlr_save = readl_relaxed(base + GITS_CTLR);
+               err = its_force_quiescent(base);
+               if (err) {
+                       pr_err("ITS@%pa: failed to quiesce: %d\n",
+                              &its->phys_base, err);
+                       writel_relaxed(its->ctlr_save, base + GITS_CTLR);
+                       goto err;
+               }
+
+               its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
+       }
+
+err:
+       if (err) {
+               list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
+                       void __iomem *base;
+
+                       if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
+                               continue;
+
+                       base = its->base;
+                       writel_relaxed(its->ctlr_save, base + GITS_CTLR);
+               }
+       }
+       spin_unlock(&its_lock);
+
+       return err;
+}
+
+static void its_restore_enable(void)
+{
+       struct its_node *its;
+       int ret;
+
+       spin_lock(&its_lock);
+       list_for_each_entry(its, &its_nodes, entry) {
+               void __iomem *base;
+               int i;
+
+               if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
+                       continue;
+
+               base = its->base;
+
+               /*
+                * Make sure that the ITS is disabled. If it fails to quiesce,
+                * don't restore it since writing to CBASER or BASER<n>
+                * registers is undefined according to the GIC v3 ITS
+                * Specification.
+                */
+               ret = its_force_quiescent(base);
+               if (ret) {
+                       pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
+                              &its->phys_base, ret);
+                       continue;
+               }
+
+               gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
+
+               /*
+                * Writing CBASER resets CREADR to 0, so make CWRITER and
+                * cmd_write line up with it.
+                */
+               its->cmd_write = its->cmd_base;
+               gits_write_cwriter(0, base + GITS_CWRITER);
+
+               /* Restore GITS_BASER from the value cache. */
+               for (i = 0; i < GITS_BASER_NR_REGS; i++) {
+                       struct its_baser *baser = &its->tables[i];
+
+                       if (!(baser->val & GITS_BASER_VALID))
+                               continue;
+
+                       its_write_baser(its, baser, baser->val);
+               }
+               writel_relaxed(its->ctlr_save, base + GITS_CTLR);
+
+               /*
+                * Reinit the collection if it's stored in the ITS. This is
+                * indicated by the col_id being less than the HCC field.
+                * CID < HCC as specified in the GIC v3 Documentation.
+                */
+               if (its->collections[smp_processor_id()].col_id <
+                   GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
+                       its_cpu_init_collection(its);
+       }
+       spin_unlock(&its_lock);
+}
+
+static struct syscore_ops its_syscore_ops = {
+       .suspend = its_save_disable,
+       .resume = its_restore_enable,
+};
+
 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
 {
        struct irq_domain *inner_domain;
@@ -3260,6 +3362,9 @@ static int __init its_probe_one(struct resource *res,
                ctlr |= GITS_CTLR_ImDe;
        writel_relaxed(ctlr, its->base + GITS_CTLR);
 
+       if (GITS_TYPER_HCC(typer))
+               its->flags |= ITS_FLAGS_SAVE_SUSPEND_STATE;
+
        err = its_init_domain(handle, its);
        if (err)
                goto out_free_tables;
@@ -3287,15 +3392,71 @@ static bool gic_rdists_supports_plpis(void)
        return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
 }
 
+static int redist_disable_lpis(void)
+{
+       void __iomem *rbase = gic_data_rdist_rd_base();
+       u64 timeout = USEC_PER_SEC;
+       u64 val;
+
+       if (!gic_rdists_supports_plpis()) {
+               pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
+               return -ENXIO;
+       }
+
+       val = readl_relaxed(rbase + GICR_CTLR);
+       if (!(val & GICR_CTLR_ENABLE_LPIS))
+               return 0;
+
+       pr_warn("CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
+               smp_processor_id());
+       add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
+
+       /* Disable LPIs */
+       val &= ~GICR_CTLR_ENABLE_LPIS;
+       writel_relaxed(val, rbase + GICR_CTLR);
+
+       /* Make sure any change to GICR_CTLR is observable by the GIC */
+       dsb(sy);
+
+       /*
+        * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
+        * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
+        * Error out if we time out waiting for RWP to clear.
+        */
+       while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
+               if (!timeout) {
+                       pr_err("CPU%d: Timeout while disabling LPIs\n",
+                              smp_processor_id());
+                       return -ETIMEDOUT;
+               }
+               udelay(1);
+               timeout--;
+       }
+
+       /*
+        * After it has been written to 1, it is IMPLEMENTATION
+        * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
+        * cleared to 0. Error out if clearing the bit failed.
+        */
+       if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
+               pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
+               return -EBUSY;
+       }
+
+       return 0;
+}
+
 int its_cpu_init(void)
 {
        if (!list_empty(&its_nodes)) {
-               if (!gic_rdists_supports_plpis()) {
-                       pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
-                       return -ENXIO;
-               }
+               int ret;
+
+               ret = redist_disable_lpis();
+               if (ret)
+                       return ret;
+
                its_cpu_init_lpis();
-               its_cpu_init_collection();
+               its_cpu_init_collections();
        }
 
        return 0;
@@ -3517,5 +3678,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
                }
        }
 
+       register_syscore_ops(&its_syscore_ops);
+
        return 0;
 }