Merge tag 'scsi-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
[muen/linux.git] / drivers / scsi / ufs / ufshcd.c
1 /*
2  * Universal Flash Storage Host controller driver Core
3  *
4  * This code is based on drivers/scsi/ufs/ufshcd.c
5  * Copyright (C) 2011-2013 Samsung India Software Operations
6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7  *
8  * Authors:
9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
10  *      Vinayak Holikatti <h.vinayak@samsung.com>
11  *
12  * This program is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU General Public License
14  * as published by the Free Software Foundation; either version 2
15  * of the License, or (at your option) any later version.
16  * See the COPYING file in the top-level directory or visit
17  * <http://www.gnu.org/licenses/gpl-2.0.html>
18  *
19  * This program is distributed in the hope that it will be useful,
20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22  * GNU General Public License for more details.
23  *
24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
25  * without warranty of any kind. You are solely responsible for
26  * determining the appropriateness of using and distributing
27  * the program and assume all risks associated with your exercise
28  * of rights with respect to the program, including but not limited
29  * to infringement of third party rights, the risks and costs of
30  * program errors, damage to or loss of data, programs or equipment,
31  * and unavailability or interruption of operations. Under no
32  * circumstances will the contributor of this Program be liable for
33  * any damages of any kind arising from your use or distribution of
34  * this program.
35  *
36  * The Linux Foundation chooses to take subject only to the GPLv2
37  * license terms, and distributes only under these terms.
38  */
39
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include "ufshcd.h"
45 #include "ufs_quirks.h"
46 #include "unipro.h"
47
48 #define CREATE_TRACE_POINTS
49 #include <trace/events/ufs.h>
50
51 #define UFSHCD_REQ_SENSE_SIZE   18
52
53 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
54                                  UTP_TASK_REQ_COMPL |\
55                                  UFSHCD_ERROR_MASK)
56 /* UIC command timeout, unit: ms */
57 #define UIC_CMD_TIMEOUT 500
58
59 /* NOP OUT retries waiting for NOP IN response */
60 #define NOP_OUT_RETRIES    10
61 /* Timeout after 30 msecs if NOP OUT hangs without response */
62 #define NOP_OUT_TIMEOUT    30 /* msecs */
63
64 /* Query request retries */
65 #define QUERY_REQ_RETRIES 3
66 /* Query request timeout */
67 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68
69 /* Task management command timeout */
70 #define TM_CMD_TIMEOUT  100 /* msecs */
71
72 /* maximum number of retries for a general UIC command  */
73 #define UFS_UIC_COMMAND_RETRIES 3
74
75 /* maximum number of link-startup retries */
76 #define DME_LINKSTARTUP_RETRIES 3
77
78 /* Maximum retries for Hibern8 enter */
79 #define UIC_HIBERN8_ENTER_RETRIES 3
80
81 /* maximum number of reset retries before giving up */
82 #define MAX_HOST_RESET_RETRIES 5
83
84 /* Expose the flag value from utp_upiu_query.value */
85 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
86
87 /* Interrupt aggregation default timeout, unit: 40us */
88 #define INT_AGGR_DEF_TO 0x02
89
90 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
91         ({                                                              \
92                 int _ret;                                               \
93                 if (_on)                                                \
94                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
95                 else                                                    \
96                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
97                 _ret;                                                   \
98         })
99
100 #define ufshcd_hex_dump(prefix_str, buf, len) \
101 print_hex_dump(KERN_ERR, prefix_str, DUMP_PREFIX_OFFSET, 16, 4, buf, len, false)
102
103 enum {
104         UFSHCD_MAX_CHANNEL      = 0,
105         UFSHCD_MAX_ID           = 1,
106         UFSHCD_CMD_PER_LUN      = 32,
107         UFSHCD_CAN_QUEUE        = 32,
108 };
109
110 /* UFSHCD states */
111 enum {
112         UFSHCD_STATE_RESET,
113         UFSHCD_STATE_ERROR,
114         UFSHCD_STATE_OPERATIONAL,
115         UFSHCD_STATE_EH_SCHEDULED,
116 };
117
118 /* UFSHCD error handling flags */
119 enum {
120         UFSHCD_EH_IN_PROGRESS = (1 << 0),
121 };
122
123 /* UFSHCD UIC layer error flags */
124 enum {
125         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
126         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
127         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
128         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
129         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
130         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
131 };
132
133 #define ufshcd_set_eh_in_progress(h) \
134         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
135 #define ufshcd_eh_in_progress(h) \
136         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
137 #define ufshcd_clear_eh_in_progress(h) \
138         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
139
140 #define ufshcd_set_ufs_dev_active(h) \
141         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
142 #define ufshcd_set_ufs_dev_sleep(h) \
143         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
144 #define ufshcd_set_ufs_dev_poweroff(h) \
145         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
146 #define ufshcd_is_ufs_dev_active(h) \
147         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
148 #define ufshcd_is_ufs_dev_sleep(h) \
149         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
150 #define ufshcd_is_ufs_dev_poweroff(h) \
151         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
152
153 static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
154         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
155         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
156         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
157         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
158         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
159         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
160 };
161
162 static inline enum ufs_dev_pwr_mode
163 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
164 {
165         return ufs_pm_lvl_states[lvl].dev_state;
166 }
167
168 static inline enum uic_link_state
169 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
170 {
171         return ufs_pm_lvl_states[lvl].link_state;
172 }
173
174 static inline enum ufs_pm_level
175 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
176                                         enum uic_link_state link_state)
177 {
178         enum ufs_pm_level lvl;
179
180         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
181                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
182                         (ufs_pm_lvl_states[lvl].link_state == link_state))
183                         return lvl;
184         }
185
186         /* if no match found, return the level 0 */
187         return UFS_PM_LVL_0;
188 }
189
190 static struct ufs_dev_fix ufs_fixups[] = {
191         /* UFS cards deviations table */
192         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
193                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
194         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
195         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
196                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
197         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
198                 UFS_DEVICE_NO_FASTAUTO),
199         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
200                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
201         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
202                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
203         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
204                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
205         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
206                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
207         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
208         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
209                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
210
211         END_FIX
212 };
213
214 static void ufshcd_tmc_handler(struct ufs_hba *hba);
215 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
216 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
217 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
218 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
219 static void ufshcd_hba_exit(struct ufs_hba *hba);
220 static int ufshcd_probe_hba(struct ufs_hba *hba);
221 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
222                                  bool skip_ref_clk);
223 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
224 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
225 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
226 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
227 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
228 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
229 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
230 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
231 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
232 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
233 static irqreturn_t ufshcd_intr(int irq, void *__hba);
234 static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
235                 struct ufs_pa_layer_attr *desired_pwr_mode);
236 static int ufshcd_change_power_mode(struct ufs_hba *hba,
237                              struct ufs_pa_layer_attr *pwr_mode);
238 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
239 {
240         return tag >= 0 && tag < hba->nutrs;
241 }
242
243 static inline int ufshcd_enable_irq(struct ufs_hba *hba)
244 {
245         int ret = 0;
246
247         if (!hba->is_irq_enabled) {
248                 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
249                                 hba);
250                 if (ret)
251                         dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
252                                 __func__, ret);
253                 hba->is_irq_enabled = true;
254         }
255
256         return ret;
257 }
258
259 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
260 {
261         if (hba->is_irq_enabled) {
262                 free_irq(hba->irq, hba);
263                 hba->is_irq_enabled = false;
264         }
265 }
266
267 /* replace non-printable or non-ASCII characters with spaces */
268 static inline void ufshcd_remove_non_printable(char *val)
269 {
270         if (!val)
271                 return;
272
273         if (*val < 0x20 || *val > 0x7e)
274                 *val = ' ';
275 }
276
277 static void ufshcd_add_command_trace(struct ufs_hba *hba,
278                 unsigned int tag, const char *str)
279 {
280         sector_t lba = -1;
281         u8 opcode = 0;
282         u32 intr, doorbell;
283         struct ufshcd_lrb *lrbp;
284         int transfer_len = -1;
285
286         if (!trace_ufshcd_command_enabled())
287                 return;
288
289         lrbp = &hba->lrb[tag];
290
291         if (lrbp->cmd) { /* data phase exists */
292                 opcode = (u8)(*lrbp->cmd->cmnd);
293                 if ((opcode == READ_10) || (opcode == WRITE_10)) {
294                         /*
295                          * Currently we only fully trace read(10) and write(10)
296                          * commands
297                          */
298                         if (lrbp->cmd->request && lrbp->cmd->request->bio)
299                                 lba =
300                                   lrbp->cmd->request->bio->bi_iter.bi_sector;
301                         transfer_len = be32_to_cpu(
302                                 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
303                 }
304         }
305
306         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
307         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
308         trace_ufshcd_command(dev_name(hba->dev), str, tag,
309                                 doorbell, transfer_len, intr, lba, opcode);
310 }
311
312 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
313 {
314         struct ufs_clk_info *clki;
315         struct list_head *head = &hba->clk_list_head;
316
317         if (list_empty(head))
318                 return;
319
320         list_for_each_entry(clki, head, list) {
321                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
322                                 clki->max_freq)
323                         dev_err(hba->dev, "clk: %s, rate: %u\n",
324                                         clki->name, clki->curr_freq);
325         }
326 }
327
328 static void ufshcd_print_uic_err_hist(struct ufs_hba *hba,
329                 struct ufs_uic_err_reg_hist *err_hist, char *err_name)
330 {
331         int i;
332
333         for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) {
334                 int p = (i + err_hist->pos - 1) % UIC_ERR_REG_HIST_LENGTH;
335
336                 if (err_hist->reg[p] == 0)
337                         continue;
338                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i,
339                         err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
340         }
341 }
342
343 static void ufshcd_print_host_regs(struct ufs_hba *hba)
344 {
345         /*
346          * hex_dump reads its data without the readl macro. This might
347          * cause inconsistency issues on some platform, as the printed
348          * values may be from cache and not the most recent value.
349          * To know whether you are looking at an un-cached version verify
350          * that IORESOURCE_MEM flag is on when xxx_get_resource() is invoked
351          * during platform/pci probe function.
352          */
353         ufshcd_hex_dump("host regs: ", hba->mmio_base, UFSHCI_REG_SPACE_SIZE);
354         dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
355                 hba->ufs_version, hba->capabilities);
356         dev_err(hba->dev,
357                 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
358                 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
359         dev_err(hba->dev,
360                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
361                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
362                 hba->ufs_stats.hibern8_exit_cnt);
363
364         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
365         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
366         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
367         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
368         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
369
370         ufshcd_print_clk_freqs(hba);
371
372         if (hba->vops && hba->vops->dbg_register_dump)
373                 hba->vops->dbg_register_dump(hba);
374 }
375
376 static
377 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
378 {
379         struct ufshcd_lrb *lrbp;
380         int prdt_length;
381         int tag;
382
383         for_each_set_bit(tag, &bitmap, hba->nutrs) {
384                 lrbp = &hba->lrb[tag];
385
386                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
387                                 tag, ktime_to_us(lrbp->issue_time_stamp));
388                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
389                                 tag, ktime_to_us(lrbp->compl_time_stamp));
390                 dev_err(hba->dev,
391                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
392                         tag, (u64)lrbp->utrd_dma_addr);
393
394                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
395                                 sizeof(struct utp_transfer_req_desc));
396                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
397                         (u64)lrbp->ucd_req_dma_addr);
398                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
399                                 sizeof(struct utp_upiu_req));
400                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
401                         (u64)lrbp->ucd_rsp_dma_addr);
402                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
403                                 sizeof(struct utp_upiu_rsp));
404
405                 prdt_length = le16_to_cpu(
406                         lrbp->utr_descriptor_ptr->prd_table_length);
407                 dev_err(hba->dev,
408                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
409                         tag, prdt_length,
410                         (u64)lrbp->ucd_prdt_dma_addr);
411
412                 if (pr_prdt)
413                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
414                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
415         }
416 }
417
418 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
419 {
420         struct utp_task_req_desc *tmrdp;
421         int tag;
422
423         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
424                 tmrdp = &hba->utmrdl_base_addr[tag];
425                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
426                 ufshcd_hex_dump("TM TRD: ", &tmrdp->header,
427                                 sizeof(struct request_desc_header));
428                 dev_err(hba->dev, "TM[%d] - Task Management Request UPIU\n",
429                                 tag);
430                 ufshcd_hex_dump("TM REQ: ", tmrdp->task_req_upiu,
431                                 sizeof(struct utp_upiu_req));
432                 dev_err(hba->dev, "TM[%d] - Task Management Response UPIU\n",
433                                 tag);
434                 ufshcd_hex_dump("TM RSP: ", tmrdp->task_rsp_upiu,
435                                 sizeof(struct utp_task_req_desc));
436         }
437 }
438
439 static void ufshcd_print_host_state(struct ufs_hba *hba)
440 {
441         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
442         dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
443                 hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
444         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
445                 hba->saved_err, hba->saved_uic_err);
446         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
447                 hba->curr_dev_pwr_mode, hba->uic_link_state);
448         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
449                 hba->pm_op_in_progress, hba->is_sys_suspended);
450         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
451                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
452         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
453         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
454                 hba->eh_flags, hba->req_abort_count);
455         dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
456                 hba->capabilities, hba->caps);
457         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
458                 hba->dev_quirks);
459 }
460
461 /**
462  * ufshcd_print_pwr_info - print power params as saved in hba
463  * power info
464  * @hba: per-adapter instance
465  */
466 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
467 {
468         static const char * const names[] = {
469                 "INVALID MODE",
470                 "FAST MODE",
471                 "SLOW_MODE",
472                 "INVALID MODE",
473                 "FASTAUTO_MODE",
474                 "SLOWAUTO_MODE",
475                 "INVALID MODE",
476         };
477
478         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
479                  __func__,
480                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
481                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
482                  names[hba->pwr_info.pwr_rx],
483                  names[hba->pwr_info.pwr_tx],
484                  hba->pwr_info.hs_rate);
485 }
486
487 /*
488  * ufshcd_wait_for_register - wait for register value to change
489  * @hba - per-adapter interface
490  * @reg - mmio register offset
491  * @mask - mask to apply to read register value
492  * @val - wait condition
493  * @interval_us - polling interval in microsecs
494  * @timeout_ms - timeout in millisecs
495  * @can_sleep - perform sleep or just spin
496  *
497  * Returns -ETIMEDOUT on error, zero on success
498  */
499 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
500                                 u32 val, unsigned long interval_us,
501                                 unsigned long timeout_ms, bool can_sleep)
502 {
503         int err = 0;
504         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
505
506         /* ignore bits that we don't intend to wait on */
507         val = val & mask;
508
509         while ((ufshcd_readl(hba, reg) & mask) != val) {
510                 if (can_sleep)
511                         usleep_range(interval_us, interval_us + 50);
512                 else
513                         udelay(interval_us);
514                 if (time_after(jiffies, timeout)) {
515                         if ((ufshcd_readl(hba, reg) & mask) != val)
516                                 err = -ETIMEDOUT;
517                         break;
518                 }
519         }
520
521         return err;
522 }
523
524 /**
525  * ufshcd_get_intr_mask - Get the interrupt bit mask
526  * @hba - Pointer to adapter instance
527  *
528  * Returns interrupt bit mask per version
529  */
530 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
531 {
532         u32 intr_mask = 0;
533
534         switch (hba->ufs_version) {
535         case UFSHCI_VERSION_10:
536                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
537                 break;
538         case UFSHCI_VERSION_11:
539         case UFSHCI_VERSION_20:
540                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
541                 break;
542         case UFSHCI_VERSION_21:
543         default:
544                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
545                 break;
546         }
547
548         return intr_mask;
549 }
550
551 /**
552  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
553  * @hba - Pointer to adapter instance
554  *
555  * Returns UFSHCI version supported by the controller
556  */
557 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
558 {
559         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
560                 return ufshcd_vops_get_ufs_hci_version(hba);
561
562         return ufshcd_readl(hba, REG_UFS_VERSION);
563 }
564
565 /**
566  * ufshcd_is_device_present - Check if any device connected to
567  *                            the host controller
568  * @hba: pointer to adapter instance
569  *
570  * Returns true if device present, false if no device detected
571  */
572 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
573 {
574         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
575                                                 DEVICE_PRESENT) ? true : false;
576 }
577
578 /**
579  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
580  * @lrb: pointer to local command reference block
581  *
582  * This function is used to get the OCS field from UTRD
583  * Returns the OCS field in the UTRD
584  */
585 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
586 {
587         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
588 }
589
590 /**
591  * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
592  * @task_req_descp: pointer to utp_task_req_desc structure
593  *
594  * This function is used to get the OCS field from UTMRD
595  * Returns the OCS field in the UTMRD
596  */
597 static inline int
598 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
599 {
600         return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
601 }
602
603 /**
604  * ufshcd_get_tm_free_slot - get a free slot for task management request
605  * @hba: per adapter instance
606  * @free_slot: pointer to variable with available slot value
607  *
608  * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
609  * Returns 0 if free slot is not available, else return 1 with tag value
610  * in @free_slot.
611  */
612 static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
613 {
614         int tag;
615         bool ret = false;
616
617         if (!free_slot)
618                 goto out;
619
620         do {
621                 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
622                 if (tag >= hba->nutmrs)
623                         goto out;
624         } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
625
626         *free_slot = tag;
627         ret = true;
628 out:
629         return ret;
630 }
631
632 static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
633 {
634         clear_bit_unlock(slot, &hba->tm_slots_in_use);
635 }
636
637 /**
638  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
639  * @hba: per adapter instance
640  * @pos: position of the bit to be cleared
641  */
642 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
643 {
644         ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
645 }
646
647 /**
648  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
649  * @hba: per adapter instance
650  * @tag: position of the bit to be cleared
651  */
652 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
653 {
654         __clear_bit(tag, &hba->outstanding_reqs);
655 }
656
657 /**
658  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
659  * @reg: Register value of host controller status
660  *
661  * Returns integer, 0 on Success and positive value if failed
662  */
663 static inline int ufshcd_get_lists_status(u32 reg)
664 {
665         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
666 }
667
668 /**
669  * ufshcd_get_uic_cmd_result - Get the UIC command result
670  * @hba: Pointer to adapter instance
671  *
672  * This function gets the result of UIC command completion
673  * Returns 0 on success, non zero value on error
674  */
675 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
676 {
677         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
678                MASK_UIC_COMMAND_RESULT;
679 }
680
681 /**
682  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
683  * @hba: Pointer to adapter instance
684  *
685  * This function gets UIC command argument3
686  * Returns 0 on success, non zero value on error
687  */
688 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
689 {
690         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
691 }
692
693 /**
694  * ufshcd_get_req_rsp - returns the TR response transaction type
695  * @ucd_rsp_ptr: pointer to response UPIU
696  */
697 static inline int
698 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
699 {
700         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
701 }
702
703 /**
704  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
705  * @ucd_rsp_ptr: pointer to response UPIU
706  *
707  * This function gets the response status and scsi_status from response UPIU
708  * Returns the response result code.
709  */
710 static inline int
711 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
712 {
713         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
714 }
715
716 /*
717  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
718  *                              from response UPIU
719  * @ucd_rsp_ptr: pointer to response UPIU
720  *
721  * Return the data segment length.
722  */
723 static inline unsigned int
724 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
725 {
726         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
727                 MASK_RSP_UPIU_DATA_SEG_LEN;
728 }
729
730 /**
731  * ufshcd_is_exception_event - Check if the device raised an exception event
732  * @ucd_rsp_ptr: pointer to response UPIU
733  *
734  * The function checks if the device raised an exception event indicated in
735  * the Device Information field of response UPIU.
736  *
737  * Returns true if exception is raised, false otherwise.
738  */
739 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
740 {
741         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
742                         MASK_RSP_EXCEPTION_EVENT ? true : false;
743 }
744
745 /**
746  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
747  * @hba: per adapter instance
748  */
749 static inline void
750 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
751 {
752         ufshcd_writel(hba, INT_AGGR_ENABLE |
753                       INT_AGGR_COUNTER_AND_TIMER_RESET,
754                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
755 }
756
757 /**
758  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
759  * @hba: per adapter instance
760  * @cnt: Interrupt aggregation counter threshold
761  * @tmout: Interrupt aggregation timeout value
762  */
763 static inline void
764 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
765 {
766         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
767                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
768                       INT_AGGR_TIMEOUT_VAL(tmout),
769                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
770 }
771
772 /**
773  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
774  * @hba: per adapter instance
775  */
776 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
777 {
778         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
779 }
780
781 /**
782  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
783  *                      When run-stop registers are set to 1, it indicates the
784  *                      host controller that it can process the requests
785  * @hba: per adapter instance
786  */
787 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
788 {
789         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
790                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
791         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
792                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
793 }
794
795 /**
796  * ufshcd_hba_start - Start controller initialization sequence
797  * @hba: per adapter instance
798  */
799 static inline void ufshcd_hba_start(struct ufs_hba *hba)
800 {
801         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
802 }
803
804 /**
805  * ufshcd_is_hba_active - Get controller state
806  * @hba: per adapter instance
807  *
808  * Returns false if controller is active, true otherwise
809  */
810 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
811 {
812         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
813                 ? false : true;
814 }
815
816 static const char *ufschd_uic_link_state_to_string(
817                         enum uic_link_state state)
818 {
819         switch (state) {
820         case UIC_LINK_OFF_STATE:        return "OFF";
821         case UIC_LINK_ACTIVE_STATE:     return "ACTIVE";
822         case UIC_LINK_HIBERN8_STATE:    return "HIBERN8";
823         default:                        return "UNKNOWN";
824         }
825 }
826
827 static const char *ufschd_ufs_dev_pwr_mode_to_string(
828                         enum ufs_dev_pwr_mode state)
829 {
830         switch (state) {
831         case UFS_ACTIVE_PWR_MODE:       return "ACTIVE";
832         case UFS_SLEEP_PWR_MODE:        return "SLEEP";
833         case UFS_POWERDOWN_PWR_MODE:    return "POWERDOWN";
834         default:                        return "UNKNOWN";
835         }
836 }
837
838 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
839 {
840         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
841         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
842             (hba->ufs_version == UFSHCI_VERSION_11))
843                 return UFS_UNIPRO_VER_1_41;
844         else
845                 return UFS_UNIPRO_VER_1_6;
846 }
847 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
848
849 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
850 {
851         /*
852          * If both host and device support UniPro ver1.6 or later, PA layer
853          * parameters tuning happens during link startup itself.
854          *
855          * We can manually tune PA layer parameters if either host or device
856          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
857          * logic simple, we will only do manual tuning if local unipro version
858          * doesn't support ver1.6 or later.
859          */
860         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
861                 return true;
862         else
863                 return false;
864 }
865
866 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
867 {
868         int ret = 0;
869         struct ufs_clk_info *clki;
870         struct list_head *head = &hba->clk_list_head;
871         ktime_t start = ktime_get();
872         bool clk_state_changed = false;
873
874         if (list_empty(head))
875                 goto out;
876
877         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
878         if (ret)
879                 return ret;
880
881         list_for_each_entry(clki, head, list) {
882                 if (!IS_ERR_OR_NULL(clki->clk)) {
883                         if (scale_up && clki->max_freq) {
884                                 if (clki->curr_freq == clki->max_freq)
885                                         continue;
886
887                                 clk_state_changed = true;
888                                 ret = clk_set_rate(clki->clk, clki->max_freq);
889                                 if (ret) {
890                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
891                                                 __func__, clki->name,
892                                                 clki->max_freq, ret);
893                                         break;
894                                 }
895                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
896                                                 "scaled up", clki->name,
897                                                 clki->curr_freq,
898                                                 clki->max_freq);
899
900                                 clki->curr_freq = clki->max_freq;
901
902                         } else if (!scale_up && clki->min_freq) {
903                                 if (clki->curr_freq == clki->min_freq)
904                                         continue;
905
906                                 clk_state_changed = true;
907                                 ret = clk_set_rate(clki->clk, clki->min_freq);
908                                 if (ret) {
909                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
910                                                 __func__, clki->name,
911                                                 clki->min_freq, ret);
912                                         break;
913                                 }
914                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
915                                                 "scaled down", clki->name,
916                                                 clki->curr_freq,
917                                                 clki->min_freq);
918                                 clki->curr_freq = clki->min_freq;
919                         }
920                 }
921                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
922                                 clki->name, clk_get_rate(clki->clk));
923         }
924
925         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
926
927 out:
928         if (clk_state_changed)
929                 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
930                         (scale_up ? "up" : "down"),
931                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
932         return ret;
933 }
934
935 /**
936  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
937  * @hba: per adapter instance
938  * @scale_up: True if scaling up and false if scaling down
939  *
940  * Returns true if scaling is required, false otherwise.
941  */
942 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
943                                                bool scale_up)
944 {
945         struct ufs_clk_info *clki;
946         struct list_head *head = &hba->clk_list_head;
947
948         if (list_empty(head))
949                 return false;
950
951         list_for_each_entry(clki, head, list) {
952                 if (!IS_ERR_OR_NULL(clki->clk)) {
953                         if (scale_up && clki->max_freq) {
954                                 if (clki->curr_freq == clki->max_freq)
955                                         continue;
956                                 return true;
957                         } else if (!scale_up && clki->min_freq) {
958                                 if (clki->curr_freq == clki->min_freq)
959                                         continue;
960                                 return true;
961                         }
962                 }
963         }
964
965         return false;
966 }
967
968 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
969                                         u64 wait_timeout_us)
970 {
971         unsigned long flags;
972         int ret = 0;
973         u32 tm_doorbell;
974         u32 tr_doorbell;
975         bool timeout = false, do_last_check = false;
976         ktime_t start;
977
978         ufshcd_hold(hba, false);
979         spin_lock_irqsave(hba->host->host_lock, flags);
980         /*
981          * Wait for all the outstanding tasks/transfer requests.
982          * Verify by checking the doorbell registers are clear.
983          */
984         start = ktime_get();
985         do {
986                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
987                         ret = -EBUSY;
988                         goto out;
989                 }
990
991                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
992                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
993                 if (!tm_doorbell && !tr_doorbell) {
994                         timeout = false;
995                         break;
996                 } else if (do_last_check) {
997                         break;
998                 }
999
1000                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1001                 schedule();
1002                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1003                     wait_timeout_us) {
1004                         timeout = true;
1005                         /*
1006                          * We might have scheduled out for long time so make
1007                          * sure to check if doorbells are cleared by this time
1008                          * or not.
1009                          */
1010                         do_last_check = true;
1011                 }
1012                 spin_lock_irqsave(hba->host->host_lock, flags);
1013         } while (tm_doorbell || tr_doorbell);
1014
1015         if (timeout) {
1016                 dev_err(hba->dev,
1017                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1018                         __func__, tm_doorbell, tr_doorbell);
1019                 ret = -EBUSY;
1020         }
1021 out:
1022         spin_unlock_irqrestore(hba->host->host_lock, flags);
1023         ufshcd_release(hba);
1024         return ret;
1025 }
1026
1027 /**
1028  * ufshcd_scale_gear - scale up/down UFS gear
1029  * @hba: per adapter instance
1030  * @scale_up: True for scaling up gear and false for scaling down
1031  *
1032  * Returns 0 for success,
1033  * Returns -EBUSY if scaling can't happen at this time
1034  * Returns non-zero for any other errors
1035  */
1036 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1037 {
1038         #define UFS_MIN_GEAR_TO_SCALE_DOWN      UFS_HS_G1
1039         int ret = 0;
1040         struct ufs_pa_layer_attr new_pwr_info;
1041
1042         if (scale_up) {
1043                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1044                        sizeof(struct ufs_pa_layer_attr));
1045         } else {
1046                 memcpy(&new_pwr_info, &hba->pwr_info,
1047                        sizeof(struct ufs_pa_layer_attr));
1048
1049                 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1050                     || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1051                         /* save the current power mode */
1052                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1053                                 &hba->pwr_info,
1054                                 sizeof(struct ufs_pa_layer_attr));
1055
1056                         /* scale down gear */
1057                         new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1058                         new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1059                 }
1060         }
1061
1062         /* check if the power mode needs to be changed or not? */
1063         ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1064
1065         if (ret)
1066                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1067                         __func__, ret,
1068                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1069                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1070
1071         return ret;
1072 }
1073
1074 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1075 {
1076         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1077         int ret = 0;
1078         /*
1079          * make sure that there are no outstanding requests when
1080          * clock scaling is in progress
1081          */
1082         scsi_block_requests(hba->host);
1083         down_write(&hba->clk_scaling_lock);
1084         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1085                 ret = -EBUSY;
1086                 up_write(&hba->clk_scaling_lock);
1087                 scsi_unblock_requests(hba->host);
1088         }
1089
1090         return ret;
1091 }
1092
1093 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1094 {
1095         up_write(&hba->clk_scaling_lock);
1096         scsi_unblock_requests(hba->host);
1097 }
1098
1099 /**
1100  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1101  * @hba: per adapter instance
1102  * @scale_up: True for scaling up and false for scalin down
1103  *
1104  * Returns 0 for success,
1105  * Returns -EBUSY if scaling can't happen at this time
1106  * Returns non-zero for any other errors
1107  */
1108 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1109 {
1110         int ret = 0;
1111
1112         /* let's not get into low power until clock scaling is completed */
1113         ufshcd_hold(hba, false);
1114
1115         ret = ufshcd_clock_scaling_prepare(hba);
1116         if (ret)
1117                 return ret;
1118
1119         /* scale down the gear before scaling down clocks */
1120         if (!scale_up) {
1121                 ret = ufshcd_scale_gear(hba, false);
1122                 if (ret)
1123                         goto out;
1124         }
1125
1126         ret = ufshcd_scale_clks(hba, scale_up);
1127         if (ret) {
1128                 if (!scale_up)
1129                         ufshcd_scale_gear(hba, true);
1130                 goto out;
1131         }
1132
1133         /* scale up the gear after scaling up clocks */
1134         if (scale_up) {
1135                 ret = ufshcd_scale_gear(hba, true);
1136                 if (ret) {
1137                         ufshcd_scale_clks(hba, false);
1138                         goto out;
1139                 }
1140         }
1141
1142         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1143
1144 out:
1145         ufshcd_clock_scaling_unprepare(hba);
1146         ufshcd_release(hba);
1147         return ret;
1148 }
1149
1150 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1151 {
1152         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1153                                            clk_scaling.suspend_work);
1154         unsigned long irq_flags;
1155
1156         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1157         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1158                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1159                 return;
1160         }
1161         hba->clk_scaling.is_suspended = true;
1162         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1163
1164         __ufshcd_suspend_clkscaling(hba);
1165 }
1166
1167 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1168 {
1169         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1170                                            clk_scaling.resume_work);
1171         unsigned long irq_flags;
1172
1173         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1174         if (!hba->clk_scaling.is_suspended) {
1175                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1176                 return;
1177         }
1178         hba->clk_scaling.is_suspended = false;
1179         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1180
1181         devfreq_resume_device(hba->devfreq);
1182 }
1183
1184 static int ufshcd_devfreq_target(struct device *dev,
1185                                 unsigned long *freq, u32 flags)
1186 {
1187         int ret = 0;
1188         struct ufs_hba *hba = dev_get_drvdata(dev);
1189         ktime_t start;
1190         bool scale_up, sched_clk_scaling_suspend_work = false;
1191         unsigned long irq_flags;
1192
1193         if (!ufshcd_is_clkscaling_supported(hba))
1194                 return -EINVAL;
1195
1196         if ((*freq > 0) && (*freq < UINT_MAX)) {
1197                 dev_err(hba->dev, "%s: invalid freq = %lu\n", __func__, *freq);
1198                 return -EINVAL;
1199         }
1200
1201         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1202         if (ufshcd_eh_in_progress(hba)) {
1203                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1204                 return 0;
1205         }
1206
1207         if (!hba->clk_scaling.active_reqs)
1208                 sched_clk_scaling_suspend_work = true;
1209
1210         scale_up = (*freq == UINT_MAX) ? true : false;
1211         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1212                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1213                 ret = 0;
1214                 goto out; /* no state change required */
1215         }
1216         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1217
1218         start = ktime_get();
1219         ret = ufshcd_devfreq_scale(hba, scale_up);
1220
1221         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1222                 (scale_up ? "up" : "down"),
1223                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1224
1225 out:
1226         if (sched_clk_scaling_suspend_work)
1227                 queue_work(hba->clk_scaling.workq,
1228                            &hba->clk_scaling.suspend_work);
1229
1230         return ret;
1231 }
1232
1233
1234 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1235                 struct devfreq_dev_status *stat)
1236 {
1237         struct ufs_hba *hba = dev_get_drvdata(dev);
1238         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1239         unsigned long flags;
1240
1241         if (!ufshcd_is_clkscaling_supported(hba))
1242                 return -EINVAL;
1243
1244         memset(stat, 0, sizeof(*stat));
1245
1246         spin_lock_irqsave(hba->host->host_lock, flags);
1247         if (!scaling->window_start_t)
1248                 goto start_window;
1249
1250         if (scaling->is_busy_started)
1251                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1252                                         scaling->busy_start_t));
1253
1254         stat->total_time = jiffies_to_usecs((long)jiffies -
1255                                 (long)scaling->window_start_t);
1256         stat->busy_time = scaling->tot_busy_t;
1257 start_window:
1258         scaling->window_start_t = jiffies;
1259         scaling->tot_busy_t = 0;
1260
1261         if (hba->outstanding_reqs) {
1262                 scaling->busy_start_t = ktime_get();
1263                 scaling->is_busy_started = true;
1264         } else {
1265                 scaling->busy_start_t = 0;
1266                 scaling->is_busy_started = false;
1267         }
1268         spin_unlock_irqrestore(hba->host->host_lock, flags);
1269         return 0;
1270 }
1271
1272 static struct devfreq_dev_profile ufs_devfreq_profile = {
1273         .polling_ms     = 100,
1274         .target         = ufshcd_devfreq_target,
1275         .get_dev_status = ufshcd_devfreq_get_dev_status,
1276 };
1277
1278 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1279 {
1280         unsigned long flags;
1281
1282         devfreq_suspend_device(hba->devfreq);
1283         spin_lock_irqsave(hba->host->host_lock, flags);
1284         hba->clk_scaling.window_start_t = 0;
1285         spin_unlock_irqrestore(hba->host->host_lock, flags);
1286 }
1287
1288 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1289 {
1290         unsigned long flags;
1291         bool suspend = false;
1292
1293         if (!ufshcd_is_clkscaling_supported(hba))
1294                 return;
1295
1296         spin_lock_irqsave(hba->host->host_lock, flags);
1297         if (!hba->clk_scaling.is_suspended) {
1298                 suspend = true;
1299                 hba->clk_scaling.is_suspended = true;
1300         }
1301         spin_unlock_irqrestore(hba->host->host_lock, flags);
1302
1303         if (suspend)
1304                 __ufshcd_suspend_clkscaling(hba);
1305 }
1306
1307 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1308 {
1309         unsigned long flags;
1310         bool resume = false;
1311
1312         if (!ufshcd_is_clkscaling_supported(hba))
1313                 return;
1314
1315         spin_lock_irqsave(hba->host->host_lock, flags);
1316         if (hba->clk_scaling.is_suspended) {
1317                 resume = true;
1318                 hba->clk_scaling.is_suspended = false;
1319         }
1320         spin_unlock_irqrestore(hba->host->host_lock, flags);
1321
1322         if (resume)
1323                 devfreq_resume_device(hba->devfreq);
1324 }
1325
1326 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1327                 struct device_attribute *attr, char *buf)
1328 {
1329         struct ufs_hba *hba = dev_get_drvdata(dev);
1330
1331         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1332 }
1333
1334 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1335                 struct device_attribute *attr, const char *buf, size_t count)
1336 {
1337         struct ufs_hba *hba = dev_get_drvdata(dev);
1338         u32 value;
1339         int err;
1340
1341         if (kstrtou32(buf, 0, &value))
1342                 return -EINVAL;
1343
1344         value = !!value;
1345         if (value == hba->clk_scaling.is_allowed)
1346                 goto out;
1347
1348         pm_runtime_get_sync(hba->dev);
1349         ufshcd_hold(hba, false);
1350
1351         cancel_work_sync(&hba->clk_scaling.suspend_work);
1352         cancel_work_sync(&hba->clk_scaling.resume_work);
1353
1354         hba->clk_scaling.is_allowed = value;
1355
1356         if (value) {
1357                 ufshcd_resume_clkscaling(hba);
1358         } else {
1359                 ufshcd_suspend_clkscaling(hba);
1360                 err = ufshcd_devfreq_scale(hba, true);
1361                 if (err)
1362                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1363                                         __func__, err);
1364         }
1365
1366         ufshcd_release(hba);
1367         pm_runtime_put_sync(hba->dev);
1368 out:
1369         return count;
1370 }
1371
1372 static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1373 {
1374         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1375         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1376         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1377         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1378         hba->clk_scaling.enable_attr.attr.mode = 0644;
1379         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1380                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1381 }
1382
1383 static void ufshcd_ungate_work(struct work_struct *work)
1384 {
1385         int ret;
1386         unsigned long flags;
1387         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1388                         clk_gating.ungate_work);
1389
1390         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1391
1392         spin_lock_irqsave(hba->host->host_lock, flags);
1393         if (hba->clk_gating.state == CLKS_ON) {
1394                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1395                 goto unblock_reqs;
1396         }
1397
1398         spin_unlock_irqrestore(hba->host->host_lock, flags);
1399         ufshcd_setup_clocks(hba, true);
1400
1401         /* Exit from hibern8 */
1402         if (ufshcd_can_hibern8_during_gating(hba)) {
1403                 /* Prevent gating in this path */
1404                 hba->clk_gating.is_suspended = true;
1405                 if (ufshcd_is_link_hibern8(hba)) {
1406                         ret = ufshcd_uic_hibern8_exit(hba);
1407                         if (ret)
1408                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1409                                         __func__, ret);
1410                         else
1411                                 ufshcd_set_link_active(hba);
1412                 }
1413                 hba->clk_gating.is_suspended = false;
1414         }
1415 unblock_reqs:
1416         scsi_unblock_requests(hba->host);
1417 }
1418
1419 /**
1420  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1421  * Also, exit from hibern8 mode and set the link as active.
1422  * @hba: per adapter instance
1423  * @async: This indicates whether caller should ungate clocks asynchronously.
1424  */
1425 int ufshcd_hold(struct ufs_hba *hba, bool async)
1426 {
1427         int rc = 0;
1428         unsigned long flags;
1429
1430         if (!ufshcd_is_clkgating_allowed(hba))
1431                 goto out;
1432         spin_lock_irqsave(hba->host->host_lock, flags);
1433         hba->clk_gating.active_reqs++;
1434
1435         if (ufshcd_eh_in_progress(hba)) {
1436                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1437                 return 0;
1438         }
1439
1440 start:
1441         switch (hba->clk_gating.state) {
1442         case CLKS_ON:
1443                 /*
1444                  * Wait for the ungate work to complete if in progress.
1445                  * Though the clocks may be in ON state, the link could
1446                  * still be in hibner8 state if hibern8 is allowed
1447                  * during clock gating.
1448                  * Make sure we exit hibern8 state also in addition to
1449                  * clocks being ON.
1450                  */
1451                 if (ufshcd_can_hibern8_during_gating(hba) &&
1452                     ufshcd_is_link_hibern8(hba)) {
1453                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1454                         flush_work(&hba->clk_gating.ungate_work);
1455                         spin_lock_irqsave(hba->host->host_lock, flags);
1456                         goto start;
1457                 }
1458                 break;
1459         case REQ_CLKS_OFF:
1460                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1461                         hba->clk_gating.state = CLKS_ON;
1462                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1463                                                 hba->clk_gating.state);
1464                         break;
1465                 }
1466                 /*
1467                  * If we are here, it means gating work is either done or
1468                  * currently running. Hence, fall through to cancel gating
1469                  * work and to enable clocks.
1470                  */
1471         case CLKS_OFF:
1472                 scsi_block_requests(hba->host);
1473                 hba->clk_gating.state = REQ_CLKS_ON;
1474                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1475                                         hba->clk_gating.state);
1476                 schedule_work(&hba->clk_gating.ungate_work);
1477                 /*
1478                  * fall through to check if we should wait for this
1479                  * work to be done or not.
1480                  */
1481         case REQ_CLKS_ON:
1482                 if (async) {
1483                         rc = -EAGAIN;
1484                         hba->clk_gating.active_reqs--;
1485                         break;
1486                 }
1487
1488                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1489                 flush_work(&hba->clk_gating.ungate_work);
1490                 /* Make sure state is CLKS_ON before returning */
1491                 spin_lock_irqsave(hba->host->host_lock, flags);
1492                 goto start;
1493         default:
1494                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1495                                 __func__, hba->clk_gating.state);
1496                 break;
1497         }
1498         spin_unlock_irqrestore(hba->host->host_lock, flags);
1499 out:
1500         return rc;
1501 }
1502 EXPORT_SYMBOL_GPL(ufshcd_hold);
1503
1504 static void ufshcd_gate_work(struct work_struct *work)
1505 {
1506         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1507                         clk_gating.gate_work.work);
1508         unsigned long flags;
1509
1510         spin_lock_irqsave(hba->host->host_lock, flags);
1511         /*
1512          * In case you are here to cancel this work the gating state
1513          * would be marked as REQ_CLKS_ON. In this case save time by
1514          * skipping the gating work and exit after changing the clock
1515          * state to CLKS_ON.
1516          */
1517         if (hba->clk_gating.is_suspended ||
1518                 (hba->clk_gating.state == REQ_CLKS_ON)) {
1519                 hba->clk_gating.state = CLKS_ON;
1520                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1521                                         hba->clk_gating.state);
1522                 goto rel_lock;
1523         }
1524
1525         if (hba->clk_gating.active_reqs
1526                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1527                 || hba->lrb_in_use || hba->outstanding_tasks
1528                 || hba->active_uic_cmd || hba->uic_async_done)
1529                 goto rel_lock;
1530
1531         spin_unlock_irqrestore(hba->host->host_lock, flags);
1532
1533         /* put the link into hibern8 mode before turning off clocks */
1534         if (ufshcd_can_hibern8_during_gating(hba)) {
1535                 if (ufshcd_uic_hibern8_enter(hba)) {
1536                         hba->clk_gating.state = CLKS_ON;
1537                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1538                                                 hba->clk_gating.state);
1539                         goto out;
1540                 }
1541                 ufshcd_set_link_hibern8(hba);
1542         }
1543
1544         if (!ufshcd_is_link_active(hba))
1545                 ufshcd_setup_clocks(hba, false);
1546         else
1547                 /* If link is active, device ref_clk can't be switched off */
1548                 __ufshcd_setup_clocks(hba, false, true);
1549
1550         /*
1551          * In case you are here to cancel this work the gating state
1552          * would be marked as REQ_CLKS_ON. In this case keep the state
1553          * as REQ_CLKS_ON which would anyway imply that clocks are off
1554          * and a request to turn them on is pending. By doing this way,
1555          * we keep the state machine in tact and this would ultimately
1556          * prevent from doing cancel work multiple times when there are
1557          * new requests arriving before the current cancel work is done.
1558          */
1559         spin_lock_irqsave(hba->host->host_lock, flags);
1560         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1561                 hba->clk_gating.state = CLKS_OFF;
1562                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1563                                         hba->clk_gating.state);
1564         }
1565 rel_lock:
1566         spin_unlock_irqrestore(hba->host->host_lock, flags);
1567 out:
1568         return;
1569 }
1570
1571 /* host lock must be held before calling this variant */
1572 static void __ufshcd_release(struct ufs_hba *hba)
1573 {
1574         if (!ufshcd_is_clkgating_allowed(hba))
1575                 return;
1576
1577         hba->clk_gating.active_reqs--;
1578
1579         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1580                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1581                 || hba->lrb_in_use || hba->outstanding_tasks
1582                 || hba->active_uic_cmd || hba->uic_async_done
1583                 || ufshcd_eh_in_progress(hba))
1584                 return;
1585
1586         hba->clk_gating.state = REQ_CLKS_OFF;
1587         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1588         schedule_delayed_work(&hba->clk_gating.gate_work,
1589                         msecs_to_jiffies(hba->clk_gating.delay_ms));
1590 }
1591
1592 void ufshcd_release(struct ufs_hba *hba)
1593 {
1594         unsigned long flags;
1595
1596         spin_lock_irqsave(hba->host->host_lock, flags);
1597         __ufshcd_release(hba);
1598         spin_unlock_irqrestore(hba->host->host_lock, flags);
1599 }
1600 EXPORT_SYMBOL_GPL(ufshcd_release);
1601
1602 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1603                 struct device_attribute *attr, char *buf)
1604 {
1605         struct ufs_hba *hba = dev_get_drvdata(dev);
1606
1607         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1608 }
1609
1610 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1611                 struct device_attribute *attr, const char *buf, size_t count)
1612 {
1613         struct ufs_hba *hba = dev_get_drvdata(dev);
1614         unsigned long flags, value;
1615
1616         if (kstrtoul(buf, 0, &value))
1617                 return -EINVAL;
1618
1619         spin_lock_irqsave(hba->host->host_lock, flags);
1620         hba->clk_gating.delay_ms = value;
1621         spin_unlock_irqrestore(hba->host->host_lock, flags);
1622         return count;
1623 }
1624
1625 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1626                 struct device_attribute *attr, char *buf)
1627 {
1628         struct ufs_hba *hba = dev_get_drvdata(dev);
1629
1630         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1631 }
1632
1633 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1634                 struct device_attribute *attr, const char *buf, size_t count)
1635 {
1636         struct ufs_hba *hba = dev_get_drvdata(dev);
1637         unsigned long flags;
1638         u32 value;
1639
1640         if (kstrtou32(buf, 0, &value))
1641                 return -EINVAL;
1642
1643         value = !!value;
1644         if (value == hba->clk_gating.is_enabled)
1645                 goto out;
1646
1647         if (value) {
1648                 ufshcd_release(hba);
1649         } else {
1650                 spin_lock_irqsave(hba->host->host_lock, flags);
1651                 hba->clk_gating.active_reqs++;
1652                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1653         }
1654
1655         hba->clk_gating.is_enabled = value;
1656 out:
1657         return count;
1658 }
1659
1660 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1661 {
1662         if (!ufshcd_is_clkgating_allowed(hba))
1663                 return;
1664
1665         hba->clk_gating.delay_ms = 150;
1666         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1667         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1668
1669         hba->clk_gating.is_enabled = true;
1670
1671         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1672         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1673         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1674         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1675         hba->clk_gating.delay_attr.attr.mode = 0644;
1676         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1677                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1678
1679         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1680         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1681         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1682         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1683         hba->clk_gating.enable_attr.attr.mode = 0644;
1684         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1685                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1686 }
1687
1688 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1689 {
1690         if (!ufshcd_is_clkgating_allowed(hba))
1691                 return;
1692         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1693         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1694         cancel_work_sync(&hba->clk_gating.ungate_work);
1695         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1696 }
1697
1698 /* Must be called with host lock acquired */
1699 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1700 {
1701         bool queue_resume_work = false;
1702
1703         if (!ufshcd_is_clkscaling_supported(hba))
1704                 return;
1705
1706         if (!hba->clk_scaling.active_reqs++)
1707                 queue_resume_work = true;
1708
1709         if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1710                 return;
1711
1712         if (queue_resume_work)
1713                 queue_work(hba->clk_scaling.workq,
1714                            &hba->clk_scaling.resume_work);
1715
1716         if (!hba->clk_scaling.window_start_t) {
1717                 hba->clk_scaling.window_start_t = jiffies;
1718                 hba->clk_scaling.tot_busy_t = 0;
1719                 hba->clk_scaling.is_busy_started = false;
1720         }
1721
1722         if (!hba->clk_scaling.is_busy_started) {
1723                 hba->clk_scaling.busy_start_t = ktime_get();
1724                 hba->clk_scaling.is_busy_started = true;
1725         }
1726 }
1727
1728 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1729 {
1730         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1731
1732         if (!ufshcd_is_clkscaling_supported(hba))
1733                 return;
1734
1735         if (!hba->outstanding_reqs && scaling->is_busy_started) {
1736                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1737                                         scaling->busy_start_t));
1738                 scaling->busy_start_t = 0;
1739                 scaling->is_busy_started = false;
1740         }
1741 }
1742 /**
1743  * ufshcd_send_command - Send SCSI or device management commands
1744  * @hba: per adapter instance
1745  * @task_tag: Task tag of the command
1746  */
1747 static inline
1748 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1749 {
1750         hba->lrb[task_tag].issue_time_stamp = ktime_get();
1751         hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
1752         ufshcd_clk_scaling_start_busy(hba);
1753         __set_bit(task_tag, &hba->outstanding_reqs);
1754         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1755         /* Make sure that doorbell is committed immediately */
1756         wmb();
1757         ufshcd_add_command_trace(hba, task_tag, "send");
1758 }
1759
1760 /**
1761  * ufshcd_copy_sense_data - Copy sense data in case of check condition
1762  * @lrb - pointer to local reference block
1763  */
1764 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1765 {
1766         int len;
1767         if (lrbp->sense_buffer &&
1768             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1769                 int len_to_copy;
1770
1771                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1772                 len_to_copy = min_t(int, RESPONSE_UPIU_SENSE_DATA_LENGTH, len);
1773
1774                 memcpy(lrbp->sense_buffer,
1775                         lrbp->ucd_rsp_ptr->sr.sense_data,
1776                         min_t(int, len_to_copy, UFSHCD_REQ_SENSE_SIZE));
1777         }
1778 }
1779
1780 /**
1781  * ufshcd_copy_query_response() - Copy the Query Response and the data
1782  * descriptor
1783  * @hba: per adapter instance
1784  * @lrb - pointer to local reference block
1785  */
1786 static
1787 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1788 {
1789         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1790
1791         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
1792
1793         /* Get the descriptor */
1794         if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
1795                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
1796                                 GENERAL_UPIU_REQUEST_SIZE;
1797                 u16 resp_len;
1798                 u16 buf_len;
1799
1800                 /* data segment length */
1801                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
1802                                                 MASK_QUERY_DATA_SEG_LEN;
1803                 buf_len = be16_to_cpu(
1804                                 hba->dev_cmd.query.request.upiu_req.length);
1805                 if (likely(buf_len >= resp_len)) {
1806                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1807                 } else {
1808                         dev_warn(hba->dev,
1809                                 "%s: Response size is bigger than buffer",
1810                                 __func__);
1811                         return -EINVAL;
1812                 }
1813         }
1814
1815         return 0;
1816 }
1817
1818 /**
1819  * ufshcd_hba_capabilities - Read controller capabilities
1820  * @hba: per adapter instance
1821  */
1822 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1823 {
1824         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1825
1826         /* nutrs and nutmrs are 0 based values */
1827         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1828         hba->nutmrs =
1829         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1830 }
1831
1832 /**
1833  * ufshcd_ready_for_uic_cmd - Check if controller is ready
1834  *                            to accept UIC commands
1835  * @hba: per adapter instance
1836  * Return true on success, else false
1837  */
1838 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1839 {
1840         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1841                 return true;
1842         else
1843                 return false;
1844 }
1845
1846 /**
1847  * ufshcd_get_upmcrs - Get the power mode change request status
1848  * @hba: Pointer to adapter instance
1849  *
1850  * This function gets the UPMCRS field of HCS register
1851  * Returns value of UPMCRS field
1852  */
1853 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1854 {
1855         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1856 }
1857
1858 /**
1859  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1860  * @hba: per adapter instance
1861  * @uic_cmd: UIC command
1862  *
1863  * Mutex must be held.
1864  */
1865 static inline void
1866 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1867 {
1868         WARN_ON(hba->active_uic_cmd);
1869
1870         hba->active_uic_cmd = uic_cmd;
1871
1872         /* Write Args */
1873         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1874         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1875         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
1876
1877         /* Write UIC Cmd */
1878         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
1879                       REG_UIC_COMMAND);
1880 }
1881
1882 /**
1883  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
1884  * @hba: per adapter instance
1885  * @uic_command: UIC command
1886  *
1887  * Must be called with mutex held.
1888  * Returns 0 only if success.
1889  */
1890 static int
1891 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1892 {
1893         int ret;
1894         unsigned long flags;
1895
1896         if (wait_for_completion_timeout(&uic_cmd->done,
1897                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
1898                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
1899         else
1900                 ret = -ETIMEDOUT;
1901
1902         spin_lock_irqsave(hba->host->host_lock, flags);
1903         hba->active_uic_cmd = NULL;
1904         spin_unlock_irqrestore(hba->host->host_lock, flags);
1905
1906         return ret;
1907 }
1908
1909 /**
1910  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1911  * @hba: per adapter instance
1912  * @uic_cmd: UIC command
1913  * @completion: initialize the completion only if this is set to true
1914  *
1915  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
1916  * with mutex held and host_lock locked.
1917  * Returns 0 only if success.
1918  */
1919 static int
1920 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
1921                       bool completion)
1922 {
1923         if (!ufshcd_ready_for_uic_cmd(hba)) {
1924                 dev_err(hba->dev,
1925                         "Controller not ready to accept UIC commands\n");
1926                 return -EIO;
1927         }
1928
1929         if (completion)
1930                 init_completion(&uic_cmd->done);
1931
1932         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
1933
1934         return 0;
1935 }
1936
1937 /**
1938  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1939  * @hba: per adapter instance
1940  * @uic_cmd: UIC command
1941  *
1942  * Returns 0 only if success.
1943  */
1944 static int
1945 ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1946 {
1947         int ret;
1948         unsigned long flags;
1949
1950         ufshcd_hold(hba, false);
1951         mutex_lock(&hba->uic_cmd_mutex);
1952         ufshcd_add_delay_before_dme_cmd(hba);
1953
1954         spin_lock_irqsave(hba->host->host_lock, flags);
1955         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
1956         spin_unlock_irqrestore(hba->host->host_lock, flags);
1957         if (!ret)
1958                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
1959
1960         mutex_unlock(&hba->uic_cmd_mutex);
1961
1962         ufshcd_release(hba);
1963         return ret;
1964 }
1965
1966 /**
1967  * ufshcd_map_sg - Map scatter-gather list to prdt
1968  * @lrbp - pointer to local reference block
1969  *
1970  * Returns 0 in case of success, non-zero value in case of failure
1971  */
1972 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1973 {
1974         struct ufshcd_sg_entry *prd_table;
1975         struct scatterlist *sg;
1976         struct scsi_cmnd *cmd;
1977         int sg_segments;
1978         int i;
1979
1980         cmd = lrbp->cmd;
1981         sg_segments = scsi_dma_map(cmd);
1982         if (sg_segments < 0)
1983                 return sg_segments;
1984
1985         if (sg_segments) {
1986                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
1987                         lrbp->utr_descriptor_ptr->prd_table_length =
1988                                 cpu_to_le16((u16)(sg_segments *
1989                                         sizeof(struct ufshcd_sg_entry)));
1990                 else
1991                         lrbp->utr_descriptor_ptr->prd_table_length =
1992                                 cpu_to_le16((u16) (sg_segments));
1993
1994                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
1995
1996                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
1997                         prd_table[i].size  =
1998                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
1999                         prd_table[i].base_addr =
2000                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2001                         prd_table[i].upper_addr =
2002                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2003                         prd_table[i].reserved = 0;
2004                 }
2005         } else {
2006                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2007         }
2008
2009         return 0;
2010 }
2011
2012 /**
2013  * ufshcd_enable_intr - enable interrupts
2014  * @hba: per adapter instance
2015  * @intrs: interrupt bits
2016  */
2017 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2018 {
2019         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2020
2021         if (hba->ufs_version == UFSHCI_VERSION_10) {
2022                 u32 rw;
2023                 rw = set & INTERRUPT_MASK_RW_VER_10;
2024                 set = rw | ((set ^ intrs) & intrs);
2025         } else {
2026                 set |= intrs;
2027         }
2028
2029         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2030 }
2031
2032 /**
2033  * ufshcd_disable_intr - disable interrupts
2034  * @hba: per adapter instance
2035  * @intrs: interrupt bits
2036  */
2037 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2038 {
2039         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2040
2041         if (hba->ufs_version == UFSHCI_VERSION_10) {
2042                 u32 rw;
2043                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2044                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2045                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2046
2047         } else {
2048                 set &= ~intrs;
2049         }
2050
2051         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2052 }
2053
2054 /**
2055  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2056  * descriptor according to request
2057  * @lrbp: pointer to local reference block
2058  * @upiu_flags: flags required in the header
2059  * @cmd_dir: requests data direction
2060  */
2061 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2062                         u32 *upiu_flags, enum dma_data_direction cmd_dir)
2063 {
2064         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2065         u32 data_direction;
2066         u32 dword_0;
2067
2068         if (cmd_dir == DMA_FROM_DEVICE) {
2069                 data_direction = UTP_DEVICE_TO_HOST;
2070                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2071         } else if (cmd_dir == DMA_TO_DEVICE) {
2072                 data_direction = UTP_HOST_TO_DEVICE;
2073                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2074         } else {
2075                 data_direction = UTP_NO_DATA_TRANSFER;
2076                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2077         }
2078
2079         dword_0 = data_direction | (lrbp->command_type
2080                                 << UPIU_COMMAND_TYPE_OFFSET);
2081         if (lrbp->intr_cmd)
2082                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2083
2084         /* Transfer request descriptor header fields */
2085         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2086         /* dword_1 is reserved, hence it is set to 0 */
2087         req_desc->header.dword_1 = 0;
2088         /*
2089          * assigning invalid value for command status. Controller
2090          * updates OCS on command completion, with the command
2091          * status
2092          */
2093         req_desc->header.dword_2 =
2094                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2095         /* dword_3 is reserved, hence it is set to 0 */
2096         req_desc->header.dword_3 = 0;
2097
2098         req_desc->prd_table_length = 0;
2099 }
2100
2101 /**
2102  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2103  * for scsi commands
2104  * @lrbp - local reference block pointer
2105  * @upiu_flags - flags
2106  */
2107 static
2108 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2109 {
2110         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2111         unsigned short cdb_len;
2112
2113         /* command descriptor fields */
2114         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2115                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2116                                 lrbp->lun, lrbp->task_tag);
2117         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2118                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2119
2120         /* Total EHS length and Data segment length will be zero */
2121         ucd_req_ptr->header.dword_2 = 0;
2122
2123         ucd_req_ptr->sc.exp_data_transfer_len =
2124                 cpu_to_be32(lrbp->cmd->sdb.length);
2125
2126         cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
2127         memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
2128         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2129
2130         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2131 }
2132
2133 /**
2134  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2135  * for query requsts
2136  * @hba: UFS hba
2137  * @lrbp: local reference block pointer
2138  * @upiu_flags: flags
2139  */
2140 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2141                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2142 {
2143         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2144         struct ufs_query *query = &hba->dev_cmd.query;
2145         u16 len = be16_to_cpu(query->request.upiu_req.length);
2146         u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
2147
2148         /* Query request header */
2149         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2150                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2151                         lrbp->lun, lrbp->task_tag);
2152         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2153                         0, query->request.query_func, 0, 0);
2154
2155         /* Data segment length only need for WRITE_DESC */
2156         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2157                 ucd_req_ptr->header.dword_2 =
2158                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2159         else
2160                 ucd_req_ptr->header.dword_2 = 0;
2161
2162         /* Copy the Query Request buffer as is */
2163         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2164                         QUERY_OSF_SIZE);
2165
2166         /* Copy the Descriptor */
2167         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2168                 memcpy(descp, query->descriptor, len);
2169
2170         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2171 }
2172
2173 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2174 {
2175         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2176
2177         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2178
2179         /* command descriptor fields */
2180         ucd_req_ptr->header.dword_0 =
2181                 UPIU_HEADER_DWORD(
2182                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2183         /* clear rest of the fields of basic header */
2184         ucd_req_ptr->header.dword_1 = 0;
2185         ucd_req_ptr->header.dword_2 = 0;
2186
2187         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2188 }
2189
2190 /**
2191  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2192  *                           for Device Management Purposes
2193  * @hba - per adapter instance
2194  * @lrb - pointer to local reference block
2195  */
2196 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2197 {
2198         u32 upiu_flags;
2199         int ret = 0;
2200
2201         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2202             (hba->ufs_version == UFSHCI_VERSION_11))
2203                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2204         else
2205                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2206
2207         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2208         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2209                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2210         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2211                 ufshcd_prepare_utp_nop_upiu(lrbp);
2212         else
2213                 ret = -EINVAL;
2214
2215         return ret;
2216 }
2217
2218 /**
2219  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2220  *                         for SCSI Purposes
2221  * @hba - per adapter instance
2222  * @lrb - pointer to local reference block
2223  */
2224 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2225 {
2226         u32 upiu_flags;
2227         int ret = 0;
2228
2229         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2230             (hba->ufs_version == UFSHCI_VERSION_11))
2231                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2232         else
2233                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2234
2235         if (likely(lrbp->cmd)) {
2236                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2237                                                 lrbp->cmd->sc_data_direction);
2238                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2239         } else {
2240                 ret = -EINVAL;
2241         }
2242
2243         return ret;
2244 }
2245
2246 /*
2247  * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN
2248  * @scsi_lun: scsi LUN id
2249  *
2250  * Returns UPIU LUN id
2251  */
2252 static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
2253 {
2254         if (scsi_is_wlun(scsi_lun))
2255                 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
2256                         | UFS_UPIU_WLUN_ID;
2257         else
2258                 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
2259 }
2260
2261 /**
2262  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2263  * @scsi_lun: UPIU W-LUN id
2264  *
2265  * Returns SCSI W-LUN id
2266  */
2267 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2268 {
2269         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2270 }
2271
2272 /**
2273  * ufshcd_queuecommand - main entry point for SCSI requests
2274  * @cmd: command from SCSI Midlayer
2275  * @done: call back function
2276  *
2277  * Returns 0 for success, non-zero in case of failure
2278  */
2279 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2280 {
2281         struct ufshcd_lrb *lrbp;
2282         struct ufs_hba *hba;
2283         unsigned long flags;
2284         int tag;
2285         int err = 0;
2286
2287         hba = shost_priv(host);
2288
2289         tag = cmd->request->tag;
2290         if (!ufshcd_valid_tag(hba, tag)) {
2291                 dev_err(hba->dev,
2292                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2293                         __func__, tag, cmd, cmd->request);
2294                 BUG();
2295         }
2296
2297         if (!down_read_trylock(&hba->clk_scaling_lock))
2298                 return SCSI_MLQUEUE_HOST_BUSY;
2299
2300         spin_lock_irqsave(hba->host->host_lock, flags);
2301         switch (hba->ufshcd_state) {
2302         case UFSHCD_STATE_OPERATIONAL:
2303                 break;
2304         case UFSHCD_STATE_EH_SCHEDULED:
2305         case UFSHCD_STATE_RESET:
2306                 err = SCSI_MLQUEUE_HOST_BUSY;
2307                 goto out_unlock;
2308         case UFSHCD_STATE_ERROR:
2309                 set_host_byte(cmd, DID_ERROR);
2310                 cmd->scsi_done(cmd);
2311                 goto out_unlock;
2312         default:
2313                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2314                                 __func__, hba->ufshcd_state);
2315                 set_host_byte(cmd, DID_BAD_TARGET);
2316                 cmd->scsi_done(cmd);
2317                 goto out_unlock;
2318         }
2319
2320         /* if error handling is in progress, don't issue commands */
2321         if (ufshcd_eh_in_progress(hba)) {
2322                 set_host_byte(cmd, DID_ERROR);
2323                 cmd->scsi_done(cmd);
2324                 goto out_unlock;
2325         }
2326         spin_unlock_irqrestore(hba->host->host_lock, flags);
2327
2328         hba->req_abort_count = 0;
2329
2330         /* acquire the tag to make sure device cmds don't use it */
2331         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
2332                 /*
2333                  * Dev manage command in progress, requeue the command.
2334                  * Requeuing the command helps in cases where the request *may*
2335                  * find different tag instead of waiting for dev manage command
2336                  * completion.
2337                  */
2338                 err = SCSI_MLQUEUE_HOST_BUSY;
2339                 goto out;
2340         }
2341
2342         err = ufshcd_hold(hba, true);
2343         if (err) {
2344                 err = SCSI_MLQUEUE_HOST_BUSY;
2345                 clear_bit_unlock(tag, &hba->lrb_in_use);
2346                 goto out;
2347         }
2348         WARN_ON(hba->clk_gating.state != CLKS_ON);
2349
2350         lrbp = &hba->lrb[tag];
2351
2352         WARN_ON(lrbp->cmd);
2353         lrbp->cmd = cmd;
2354         lrbp->sense_bufflen = UFSHCD_REQ_SENSE_SIZE;
2355         lrbp->sense_buffer = cmd->sense_buffer;
2356         lrbp->task_tag = tag;
2357         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2358         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2359         lrbp->req_abort_skip = false;
2360
2361         ufshcd_comp_scsi_upiu(hba, lrbp);
2362
2363         err = ufshcd_map_sg(hba, lrbp);
2364         if (err) {
2365                 lrbp->cmd = NULL;
2366                 clear_bit_unlock(tag, &hba->lrb_in_use);
2367                 goto out;
2368         }
2369         /* Make sure descriptors are ready before ringing the doorbell */
2370         wmb();
2371
2372         /* issue command to the controller */
2373         spin_lock_irqsave(hba->host->host_lock, flags);
2374         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2375         ufshcd_send_command(hba, tag);
2376 out_unlock:
2377         spin_unlock_irqrestore(hba->host->host_lock, flags);
2378 out:
2379         up_read(&hba->clk_scaling_lock);
2380         return err;
2381 }
2382
2383 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2384                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2385 {
2386         lrbp->cmd = NULL;
2387         lrbp->sense_bufflen = 0;
2388         lrbp->sense_buffer = NULL;
2389         lrbp->task_tag = tag;
2390         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2391         lrbp->intr_cmd = true; /* No interrupt aggregation */
2392         hba->dev_cmd.type = cmd_type;
2393
2394         return ufshcd_comp_devman_upiu(hba, lrbp);
2395 }
2396
2397 static int
2398 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2399 {
2400         int err = 0;
2401         unsigned long flags;
2402         u32 mask = 1 << tag;
2403
2404         /* clear outstanding transaction before retry */
2405         spin_lock_irqsave(hba->host->host_lock, flags);
2406         ufshcd_utrl_clear(hba, tag);
2407         spin_unlock_irqrestore(hba->host->host_lock, flags);
2408
2409         /*
2410          * wait for for h/w to clear corresponding bit in door-bell.
2411          * max. wait is 1 sec.
2412          */
2413         err = ufshcd_wait_for_register(hba,
2414                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2415                         mask, ~mask, 1000, 1000, true);
2416
2417         return err;
2418 }
2419
2420 static int
2421 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2422 {
2423         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2424
2425         /* Get the UPIU response */
2426         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2427                                 UPIU_RSP_CODE_OFFSET;
2428         return query_res->response;
2429 }
2430
2431 /**
2432  * ufshcd_dev_cmd_completion() - handles device management command responses
2433  * @hba: per adapter instance
2434  * @lrbp: pointer to local reference block
2435  */
2436 static int
2437 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2438 {
2439         int resp;
2440         int err = 0;
2441
2442         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2443         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2444
2445         switch (resp) {
2446         case UPIU_TRANSACTION_NOP_IN:
2447                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2448                         err = -EINVAL;
2449                         dev_err(hba->dev, "%s: unexpected response %x\n",
2450                                         __func__, resp);
2451                 }
2452                 break;
2453         case UPIU_TRANSACTION_QUERY_RSP:
2454                 err = ufshcd_check_query_response(hba, lrbp);
2455                 if (!err)
2456                         err = ufshcd_copy_query_response(hba, lrbp);
2457                 break;
2458         case UPIU_TRANSACTION_REJECT_UPIU:
2459                 /* TODO: handle Reject UPIU Response */
2460                 err = -EPERM;
2461                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2462                                 __func__);
2463                 break;
2464         default:
2465                 err = -EINVAL;
2466                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2467                                 __func__, resp);
2468                 break;
2469         }
2470
2471         return err;
2472 }
2473
2474 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2475                 struct ufshcd_lrb *lrbp, int max_timeout)
2476 {
2477         int err = 0;
2478         unsigned long time_left;
2479         unsigned long flags;
2480
2481         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2482                         msecs_to_jiffies(max_timeout));
2483
2484         /* Make sure descriptors are ready before ringing the doorbell */
2485         wmb();
2486         spin_lock_irqsave(hba->host->host_lock, flags);
2487         hba->dev_cmd.complete = NULL;
2488         if (likely(time_left)) {
2489                 err = ufshcd_get_tr_ocs(lrbp);
2490                 if (!err)
2491                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2492         }
2493         spin_unlock_irqrestore(hba->host->host_lock, flags);
2494
2495         if (!time_left) {
2496                 err = -ETIMEDOUT;
2497                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2498                         __func__, lrbp->task_tag);
2499                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2500                         /* successfully cleared the command, retry if needed */
2501                         err = -EAGAIN;
2502                 /*
2503                  * in case of an error, after clearing the doorbell,
2504                  * we also need to clear the outstanding_request
2505                  * field in hba
2506                  */
2507                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2508         }
2509
2510         return err;
2511 }
2512
2513 /**
2514  * ufshcd_get_dev_cmd_tag - Get device management command tag
2515  * @hba: per-adapter instance
2516  * @tag: pointer to variable with available slot value
2517  *
2518  * Get a free slot and lock it until device management command
2519  * completes.
2520  *
2521  * Returns false if free slot is unavailable for locking, else
2522  * return true with tag value in @tag.
2523  */
2524 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
2525 {
2526         int tag;
2527         bool ret = false;
2528         unsigned long tmp;
2529
2530         if (!tag_out)
2531                 goto out;
2532
2533         do {
2534                 tmp = ~hba->lrb_in_use;
2535                 tag = find_last_bit(&tmp, hba->nutrs);
2536                 if (tag >= hba->nutrs)
2537                         goto out;
2538         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
2539
2540         *tag_out = tag;
2541         ret = true;
2542 out:
2543         return ret;
2544 }
2545
2546 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
2547 {
2548         clear_bit_unlock(tag, &hba->lrb_in_use);
2549 }
2550
2551 /**
2552  * ufshcd_exec_dev_cmd - API for sending device management requests
2553  * @hba - UFS hba
2554  * @cmd_type - specifies the type (NOP, Query...)
2555  * @timeout - time in seconds
2556  *
2557  * NOTE: Since there is only one available tag for device management commands,
2558  * it is expected you hold the hba->dev_cmd.lock mutex.
2559  */
2560 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2561                 enum dev_cmd_type cmd_type, int timeout)
2562 {
2563         struct ufshcd_lrb *lrbp;
2564         int err;
2565         int tag;
2566         struct completion wait;
2567         unsigned long flags;
2568
2569         down_read(&hba->clk_scaling_lock);
2570
2571         /*
2572          * Get free slot, sleep if slots are unavailable.
2573          * Even though we use wait_event() which sleeps indefinitely,
2574          * the maximum wait time is bounded by SCSI request timeout.
2575          */
2576         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
2577
2578         init_completion(&wait);
2579         lrbp = &hba->lrb[tag];
2580         WARN_ON(lrbp->cmd);
2581         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2582         if (unlikely(err))
2583                 goto out_put_tag;
2584
2585         hba->dev_cmd.complete = &wait;
2586
2587         /* Make sure descriptors are ready before ringing the doorbell */
2588         wmb();
2589         spin_lock_irqsave(hba->host->host_lock, flags);
2590         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2591         ufshcd_send_command(hba, tag);
2592         spin_unlock_irqrestore(hba->host->host_lock, flags);
2593
2594         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2595
2596 out_put_tag:
2597         ufshcd_put_dev_cmd_tag(hba, tag);
2598         wake_up(&hba->dev_cmd.tag_wq);
2599         up_read(&hba->clk_scaling_lock);
2600         return err;
2601 }
2602
2603 /**
2604  * ufshcd_init_query() - init the query response and request parameters
2605  * @hba: per-adapter instance
2606  * @request: address of the request pointer to be initialized
2607  * @response: address of the response pointer to be initialized
2608  * @opcode: operation to perform
2609  * @idn: flag idn to access
2610  * @index: LU number to access
2611  * @selector: query/flag/descriptor further identification
2612  */
2613 static inline void ufshcd_init_query(struct ufs_hba *hba,
2614                 struct ufs_query_req **request, struct ufs_query_res **response,
2615                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2616 {
2617         *request = &hba->dev_cmd.query.request;
2618         *response = &hba->dev_cmd.query.response;
2619         memset(*request, 0, sizeof(struct ufs_query_req));
2620         memset(*response, 0, sizeof(struct ufs_query_res));
2621         (*request)->upiu_req.opcode = opcode;
2622         (*request)->upiu_req.idn = idn;
2623         (*request)->upiu_req.index = index;
2624         (*request)->upiu_req.selector = selector;
2625 }
2626
2627 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2628         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2629 {
2630         int ret;
2631         int retries;
2632
2633         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2634                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2635                 if (ret)
2636                         dev_dbg(hba->dev,
2637                                 "%s: failed with error %d, retries %d\n",
2638                                 __func__, ret, retries);
2639                 else
2640                         break;
2641         }
2642
2643         if (ret)
2644                 dev_err(hba->dev,
2645                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2646                         __func__, opcode, idn, ret, retries);
2647         return ret;
2648 }
2649
2650 /**
2651  * ufshcd_query_flag() - API function for sending flag query requests
2652  * hba: per-adapter instance
2653  * query_opcode: flag query to perform
2654  * idn: flag idn to access
2655  * flag_res: the flag value after the query request completes
2656  *
2657  * Returns 0 for success, non-zero in case of failure
2658  */
2659 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2660                         enum flag_idn idn, bool *flag_res)
2661 {
2662         struct ufs_query_req *request = NULL;
2663         struct ufs_query_res *response = NULL;
2664         int err, index = 0, selector = 0;
2665         int timeout = QUERY_REQ_TIMEOUT;
2666
2667         BUG_ON(!hba);
2668
2669         ufshcd_hold(hba, false);
2670         mutex_lock(&hba->dev_cmd.lock);
2671         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2672                         selector);
2673
2674         switch (opcode) {
2675         case UPIU_QUERY_OPCODE_SET_FLAG:
2676         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2677         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2678                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2679                 break;
2680         case UPIU_QUERY_OPCODE_READ_FLAG:
2681                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2682                 if (!flag_res) {
2683                         /* No dummy reads */
2684                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
2685                                         __func__);
2686                         err = -EINVAL;
2687                         goto out_unlock;
2688                 }
2689                 break;
2690         default:
2691                 dev_err(hba->dev,
2692                         "%s: Expected query flag opcode but got = %d\n",
2693                         __func__, opcode);
2694                 err = -EINVAL;
2695                 goto out_unlock;
2696         }
2697
2698         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2699
2700         if (err) {
2701                 dev_err(hba->dev,
2702                         "%s: Sending flag query for idn %d failed, err = %d\n",
2703                         __func__, idn, err);
2704                 goto out_unlock;
2705         }
2706
2707         if (flag_res)
2708                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2709                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2710
2711 out_unlock:
2712         mutex_unlock(&hba->dev_cmd.lock);
2713         ufshcd_release(hba);
2714         return err;
2715 }
2716
2717 /**
2718  * ufshcd_query_attr - API function for sending attribute requests
2719  * hba: per-adapter instance
2720  * opcode: attribute opcode
2721  * idn: attribute idn to access
2722  * index: index field
2723  * selector: selector field
2724  * attr_val: the attribute value after the query request completes
2725  *
2726  * Returns 0 for success, non-zero in case of failure
2727 */
2728 static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2729                         enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2730 {
2731         struct ufs_query_req *request = NULL;
2732         struct ufs_query_res *response = NULL;
2733         int err;
2734
2735         BUG_ON(!hba);
2736
2737         ufshcd_hold(hba, false);
2738         if (!attr_val) {
2739                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2740                                 __func__, opcode);
2741                 err = -EINVAL;
2742                 goto out;
2743         }
2744
2745         mutex_lock(&hba->dev_cmd.lock);
2746         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2747                         selector);
2748
2749         switch (opcode) {
2750         case UPIU_QUERY_OPCODE_WRITE_ATTR:
2751                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2752                 request->upiu_req.value = cpu_to_be32(*attr_val);
2753                 break;
2754         case UPIU_QUERY_OPCODE_READ_ATTR:
2755                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2756                 break;
2757         default:
2758                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2759                                 __func__, opcode);
2760                 err = -EINVAL;
2761                 goto out_unlock;
2762         }
2763
2764         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2765
2766         if (err) {
2767                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2768                                 __func__, opcode, idn, index, err);
2769                 goto out_unlock;
2770         }
2771
2772         *attr_val = be32_to_cpu(response->upiu_res.value);
2773
2774 out_unlock:
2775         mutex_unlock(&hba->dev_cmd.lock);
2776 out:
2777         ufshcd_release(hba);
2778         return err;
2779 }
2780
2781 /**
2782  * ufshcd_query_attr_retry() - API function for sending query
2783  * attribute with retries
2784  * @hba: per-adapter instance
2785  * @opcode: attribute opcode
2786  * @idn: attribute idn to access
2787  * @index: index field
2788  * @selector: selector field
2789  * @attr_val: the attribute value after the query request
2790  * completes
2791  *
2792  * Returns 0 for success, non-zero in case of failure
2793 */
2794 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2795         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2796         u32 *attr_val)
2797 {
2798         int ret = 0;
2799         u32 retries;
2800
2801          for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2802                 ret = ufshcd_query_attr(hba, opcode, idn, index,
2803                                                 selector, attr_val);
2804                 if (ret)
2805                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2806                                 __func__, ret, retries);
2807                 else
2808                         break;
2809         }
2810
2811         if (ret)
2812                 dev_err(hba->dev,
2813                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2814                         __func__, idn, ret, QUERY_REQ_RETRIES);
2815         return ret;
2816 }
2817
2818 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
2819                         enum query_opcode opcode, enum desc_idn idn, u8 index,
2820                         u8 selector, u8 *desc_buf, int *buf_len)
2821 {
2822         struct ufs_query_req *request = NULL;
2823         struct ufs_query_res *response = NULL;
2824         int err;
2825
2826         BUG_ON(!hba);
2827
2828         ufshcd_hold(hba, false);
2829         if (!desc_buf) {
2830                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2831                                 __func__, opcode);
2832                 err = -EINVAL;
2833                 goto out;
2834         }
2835
2836         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
2837                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2838                                 __func__, *buf_len);
2839                 err = -EINVAL;
2840                 goto out;
2841         }
2842
2843         mutex_lock(&hba->dev_cmd.lock);
2844         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2845                         selector);
2846         hba->dev_cmd.query.descriptor = desc_buf;
2847         request->upiu_req.length = cpu_to_be16(*buf_len);
2848
2849         switch (opcode) {
2850         case UPIU_QUERY_OPCODE_WRITE_DESC:
2851                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2852                 break;
2853         case UPIU_QUERY_OPCODE_READ_DESC:
2854                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2855                 break;
2856         default:
2857                 dev_err(hba->dev,
2858                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2859                                 __func__, opcode);
2860                 err = -EINVAL;
2861                 goto out_unlock;
2862         }
2863
2864         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2865
2866         if (err) {
2867                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2868                                 __func__, opcode, idn, index, err);
2869                 goto out_unlock;
2870         }
2871
2872         hba->dev_cmd.query.descriptor = NULL;
2873         *buf_len = be16_to_cpu(response->upiu_res.length);
2874
2875 out_unlock:
2876         mutex_unlock(&hba->dev_cmd.lock);
2877 out:
2878         ufshcd_release(hba);
2879         return err;
2880 }
2881
2882 /**
2883  * ufshcd_query_descriptor_retry - API function for sending descriptor
2884  * requests
2885  * hba: per-adapter instance
2886  * opcode: attribute opcode
2887  * idn: attribute idn to access
2888  * index: index field
2889  * selector: selector field
2890  * desc_buf: the buffer that contains the descriptor
2891  * buf_len: length parameter passed to the device
2892  *
2893  * Returns 0 for success, non-zero in case of failure.
2894  * The buf_len parameter will contain, on return, the length parameter
2895  * received on the response.
2896  */
2897 static int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
2898                                          enum query_opcode opcode,
2899                                          enum desc_idn idn, u8 index,
2900                                          u8 selector,
2901                                          u8 *desc_buf, int *buf_len)
2902 {
2903         int err;
2904         int retries;
2905
2906         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2907                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
2908                                                 selector, desc_buf, buf_len);
2909                 if (!err || err == -EINVAL)
2910                         break;
2911         }
2912
2913         return err;
2914 }
2915
2916 /**
2917  * ufshcd_read_desc_length - read the specified descriptor length from header
2918  * @hba: Pointer to adapter instance
2919  * @desc_id: descriptor idn value
2920  * @desc_index: descriptor index
2921  * @desc_length: pointer to variable to read the length of descriptor
2922  *
2923  * Return 0 in case of success, non-zero otherwise
2924  */
2925 static int ufshcd_read_desc_length(struct ufs_hba *hba,
2926         enum desc_idn desc_id,
2927         int desc_index,
2928         int *desc_length)
2929 {
2930         int ret;
2931         u8 header[QUERY_DESC_HDR_SIZE];
2932         int header_len = QUERY_DESC_HDR_SIZE;
2933
2934         if (desc_id >= QUERY_DESC_IDN_MAX)
2935                 return -EINVAL;
2936
2937         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2938                                         desc_id, desc_index, 0, header,
2939                                         &header_len);
2940
2941         if (ret) {
2942                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
2943                         __func__, desc_id);
2944                 return ret;
2945         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
2946                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
2947                         __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
2948                         desc_id);
2949                 ret = -EINVAL;
2950         }
2951
2952         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
2953         return ret;
2954
2955 }
2956
2957 /**
2958  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
2959  * @hba: Pointer to adapter instance
2960  * @desc_id: descriptor idn value
2961  * @desc_len: mapped desc length (out)
2962  *
2963  * Return 0 in case of success, non-zero otherwise
2964  */
2965 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
2966         enum desc_idn desc_id, int *desc_len)
2967 {
2968         switch (desc_id) {
2969         case QUERY_DESC_IDN_DEVICE:
2970                 *desc_len = hba->desc_size.dev_desc;
2971                 break;
2972         case QUERY_DESC_IDN_POWER:
2973                 *desc_len = hba->desc_size.pwr_desc;
2974                 break;
2975         case QUERY_DESC_IDN_GEOMETRY:
2976                 *desc_len = hba->desc_size.geom_desc;
2977                 break;
2978         case QUERY_DESC_IDN_CONFIGURATION:
2979                 *desc_len = hba->desc_size.conf_desc;
2980                 break;
2981         case QUERY_DESC_IDN_UNIT:
2982                 *desc_len = hba->desc_size.unit_desc;
2983                 break;
2984         case QUERY_DESC_IDN_INTERCONNECT:
2985                 *desc_len = hba->desc_size.interc_desc;
2986                 break;
2987         case QUERY_DESC_IDN_STRING:
2988                 *desc_len = QUERY_DESC_MAX_SIZE;
2989                 break;
2990         case QUERY_DESC_IDN_RFU_0:
2991         case QUERY_DESC_IDN_RFU_1:
2992                 *desc_len = 0;
2993                 break;
2994         default:
2995                 *desc_len = 0;
2996                 return -EINVAL;
2997         }
2998         return 0;
2999 }
3000 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3001
3002 /**
3003  * ufshcd_read_desc_param - read the specified descriptor parameter
3004  * @hba: Pointer to adapter instance
3005  * @desc_id: descriptor idn value
3006  * @desc_index: descriptor index
3007  * @param_offset: offset of the parameter to read
3008  * @param_read_buf: pointer to buffer where parameter would be read
3009  * @param_size: sizeof(param_read_buf)
3010  *
3011  * Return 0 in case of success, non-zero otherwise
3012  */
3013 static int ufshcd_read_desc_param(struct ufs_hba *hba,
3014                                   enum desc_idn desc_id,
3015                                   int desc_index,
3016                                   u8 param_offset,
3017                                   u8 *param_read_buf,
3018                                   u8 param_size)
3019 {
3020         int ret;
3021         u8 *desc_buf;
3022         int buff_len;
3023         bool is_kmalloc = true;
3024
3025         /* Safety check */
3026         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3027                 return -EINVAL;
3028
3029         /* Get the max length of descriptor from structure filled up at probe
3030          * time.
3031          */
3032         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3033
3034         /* Sanity checks */
3035         if (ret || !buff_len) {
3036                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3037                         __func__);
3038                 return ret;
3039         }
3040
3041         /* Check whether we need temp memory */
3042         if (param_offset != 0 || param_size < buff_len) {
3043                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3044                 if (!desc_buf)
3045                         return -ENOMEM;
3046         } else {
3047                 desc_buf = param_read_buf;
3048                 is_kmalloc = false;
3049         }
3050
3051         /* Request for full descriptor */
3052         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3053                                         desc_id, desc_index, 0,
3054                                         desc_buf, &buff_len);
3055
3056         if (ret) {
3057                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3058                         __func__, desc_id, desc_index, param_offset, ret);
3059                 goto out;
3060         }
3061
3062         /* Sanity check */
3063         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3064                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3065                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3066                 ret = -EINVAL;
3067                 goto out;
3068         }
3069
3070         /* Check wherher we will not copy more data, than available */
3071         if (is_kmalloc && param_size > buff_len)
3072                 param_size = buff_len;
3073
3074         if (is_kmalloc)
3075                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3076 out:
3077         if (is_kmalloc)
3078                 kfree(desc_buf);
3079         return ret;
3080 }
3081
3082 static inline int ufshcd_read_desc(struct ufs_hba *hba,
3083                                    enum desc_idn desc_id,
3084                                    int desc_index,
3085                                    u8 *buf,
3086                                    u32 size)
3087 {
3088         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3089 }
3090
3091 static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3092                                          u8 *buf,
3093                                          u32 size)
3094 {
3095         return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
3096 }
3097
3098 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
3099 {
3100         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3101 }
3102
3103 /**
3104  * ufshcd_read_string_desc - read string descriptor
3105  * @hba: pointer to adapter instance
3106  * @desc_index: descriptor index
3107  * @buf: pointer to buffer where descriptor would be read
3108  * @size: size of buf
3109  * @ascii: if true convert from unicode to ascii characters
3110  *
3111  * Return 0 in case of success, non-zero otherwise
3112  */
3113 #define ASCII_STD true
3114 static int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
3115                                    u8 *buf, u32 size, bool ascii)
3116 {
3117         int err = 0;
3118
3119         err = ufshcd_read_desc(hba,
3120                                 QUERY_DESC_IDN_STRING, desc_index, buf, size);
3121
3122         if (err) {
3123                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
3124                         __func__, QUERY_REQ_RETRIES, err);
3125                 goto out;
3126         }
3127
3128         if (ascii) {
3129                 int desc_len;
3130                 int ascii_len;
3131                 int i;
3132                 char *buff_ascii;
3133
3134                 desc_len = buf[0];
3135                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3136                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3137                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
3138                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
3139                                         __func__);
3140                         err = -ENOMEM;
3141                         goto out;
3142                 }
3143
3144                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
3145                 if (!buff_ascii) {
3146                         err = -ENOMEM;
3147                         goto out;
3148                 }
3149
3150                 /*
3151                  * the descriptor contains string in UTF16 format
3152                  * we need to convert to utf-8 so it can be displayed
3153                  */
3154                 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
3155                                 desc_len - QUERY_DESC_HDR_SIZE,
3156                                 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
3157
3158                 /* replace non-printable or non-ASCII characters with spaces */
3159                 for (i = 0; i < ascii_len; i++)
3160                         ufshcd_remove_non_printable(&buff_ascii[i]);
3161
3162                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
3163                                 size - QUERY_DESC_HDR_SIZE);
3164                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
3165                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
3166                 kfree(buff_ascii);
3167         }
3168 out:
3169         return err;
3170 }
3171
3172 /**
3173  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3174  * @hba: Pointer to adapter instance
3175  * @lun: lun id
3176  * @param_offset: offset of the parameter to read
3177  * @param_read_buf: pointer to buffer where parameter would be read
3178  * @param_size: sizeof(param_read_buf)
3179  *
3180  * Return 0 in case of success, non-zero otherwise
3181  */
3182 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3183                                               int lun,
3184                                               enum unit_desc_param param_offset,
3185                                               u8 *param_read_buf,
3186                                               u32 param_size)
3187 {
3188         /*
3189          * Unit descriptors are only available for general purpose LUs (LUN id
3190          * from 0 to 7) and RPMB Well known LU.
3191          */
3192         if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN))
3193                 return -EOPNOTSUPP;
3194
3195         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3196                                       param_offset, param_read_buf, param_size);
3197 }
3198
3199 /**
3200  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3201  * @hba: per adapter instance
3202  *
3203  * 1. Allocate DMA memory for Command Descriptor array
3204  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3205  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3206  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3207  *      (UTMRDL)
3208  * 4. Allocate memory for local reference block(lrb).
3209  *
3210  * Returns 0 for success, non-zero in case of failure
3211  */
3212 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3213 {
3214         size_t utmrdl_size, utrdl_size, ucdl_size;
3215
3216         /* Allocate memory for UTP command descriptors */
3217         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3218         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3219                                                   ucdl_size,
3220                                                   &hba->ucdl_dma_addr,
3221                                                   GFP_KERNEL);
3222
3223         /*
3224          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3225          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3226          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3227          * be aligned to 128 bytes as well
3228          */
3229         if (!hba->ucdl_base_addr ||
3230             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3231                 dev_err(hba->dev,
3232                         "Command Descriptor Memory allocation failed\n");
3233                 goto out;
3234         }
3235
3236         /*
3237          * Allocate memory for UTP Transfer descriptors
3238          * UFSHCI requires 1024 byte alignment of UTRD
3239          */
3240         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3241         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3242                                                    utrdl_size,
3243                                                    &hba->utrdl_dma_addr,
3244                                                    GFP_KERNEL);
3245         if (!hba->utrdl_base_addr ||
3246             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3247                 dev_err(hba->dev,
3248                         "Transfer Descriptor Memory allocation failed\n");
3249                 goto out;
3250         }
3251
3252         /*
3253          * Allocate memory for UTP Task Management descriptors
3254          * UFSHCI requires 1024 byte alignment of UTMRD
3255          */
3256         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3257         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3258                                                     utmrdl_size,
3259                                                     &hba->utmrdl_dma_addr,
3260                                                     GFP_KERNEL);
3261         if (!hba->utmrdl_base_addr ||
3262             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3263                 dev_err(hba->dev,
3264                 "Task Management Descriptor Memory allocation failed\n");
3265                 goto out;
3266         }
3267
3268         /* Allocate memory for local reference block */
3269         hba->lrb = devm_kzalloc(hba->dev,
3270                                 hba->nutrs * sizeof(struct ufshcd_lrb),
3271                                 GFP_KERNEL);
3272         if (!hba->lrb) {
3273                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3274                 goto out;
3275         }
3276         return 0;
3277 out:
3278         return -ENOMEM;
3279 }
3280
3281 /**
3282  * ufshcd_host_memory_configure - configure local reference block with
3283  *                              memory offsets
3284  * @hba: per adapter instance
3285  *
3286  * Configure Host memory space
3287  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3288  * address.
3289  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3290  * and PRDT offset.
3291  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3292  * into local reference block.
3293  */
3294 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3295 {
3296         struct utp_transfer_cmd_desc *cmd_descp;
3297         struct utp_transfer_req_desc *utrdlp;
3298         dma_addr_t cmd_desc_dma_addr;
3299         dma_addr_t cmd_desc_element_addr;
3300         u16 response_offset;
3301         u16 prdt_offset;
3302         int cmd_desc_size;
3303         int i;
3304
3305         utrdlp = hba->utrdl_base_addr;
3306         cmd_descp = hba->ucdl_base_addr;
3307
3308         response_offset =
3309                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3310         prdt_offset =
3311                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3312
3313         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3314         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3315
3316         for (i = 0; i < hba->nutrs; i++) {
3317                 /* Configure UTRD with command descriptor base address */
3318                 cmd_desc_element_addr =
3319                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3320                 utrdlp[i].command_desc_base_addr_lo =
3321                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3322                 utrdlp[i].command_desc_base_addr_hi =
3323                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3324
3325                 /* Response upiu and prdt offset should be in double words */
3326                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3327                         utrdlp[i].response_upiu_offset =
3328                                 cpu_to_le16(response_offset);
3329                         utrdlp[i].prd_table_offset =
3330                                 cpu_to_le16(prdt_offset);
3331                         utrdlp[i].response_upiu_length =
3332                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3333                 } else {
3334                         utrdlp[i].response_upiu_offset =
3335                                 cpu_to_le16((response_offset >> 2));
3336                         utrdlp[i].prd_table_offset =
3337                                 cpu_to_le16((prdt_offset >> 2));
3338                         utrdlp[i].response_upiu_length =
3339                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3340                 }
3341
3342                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
3343                 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3344                                 (i * sizeof(struct utp_transfer_req_desc));
3345                 hba->lrb[i].ucd_req_ptr =
3346                         (struct utp_upiu_req *)(cmd_descp + i);
3347                 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
3348                 hba->lrb[i].ucd_rsp_ptr =
3349                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
3350                 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3351                                 response_offset;
3352                 hba->lrb[i].ucd_prdt_ptr =
3353                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
3354                 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3355                                 prdt_offset;
3356         }
3357 }
3358
3359 /**
3360  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3361  * @hba: per adapter instance
3362  *
3363  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3364  * in order to initialize the Unipro link startup procedure.
3365  * Once the Unipro links are up, the device connected to the controller
3366  * is detected.
3367  *
3368  * Returns 0 on success, non-zero value on failure
3369  */
3370 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3371 {
3372         struct uic_command uic_cmd = {0};
3373         int ret;
3374
3375         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3376
3377         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3378         if (ret)
3379                 dev_dbg(hba->dev,
3380                         "dme-link-startup: error code %d\n", ret);
3381         return ret;
3382 }
3383
3384 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3385 {
3386         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3387         unsigned long min_sleep_time_us;
3388
3389         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3390                 return;
3391
3392         /*
3393          * last_dme_cmd_tstamp will be 0 only for 1st call to
3394          * this function
3395          */
3396         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3397                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3398         } else {
3399                 unsigned long delta =
3400                         (unsigned long) ktime_to_us(
3401                                 ktime_sub(ktime_get(),
3402                                 hba->last_dme_cmd_tstamp));
3403
3404                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3405                         min_sleep_time_us =
3406                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3407                 else
3408                         return; /* no more delay required */
3409         }
3410
3411         /* allow sleep for extra 50us if needed */
3412         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3413 }
3414
3415 /**
3416  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3417  * @hba: per adapter instance
3418  * @attr_sel: uic command argument1
3419  * @attr_set: attribute set type as uic command argument2
3420  * @mib_val: setting value as uic command argument3
3421  * @peer: indicate whether peer or local
3422  *
3423  * Returns 0 on success, non-zero value on failure
3424  */
3425 in