source: src/linux/universal/linux-4.9/drivers/scsi/ufs/ufshcd.c @ 31859

Last change on this file since 31859 was 31859, checked in by brainslayer, 2 months ago

kernel update

File size: 174.9 KB
Line 
1/*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 *      Santosh Yaraganavi <santosh.sy@samsung.com>
10 *      Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
39
40#include <linux/async.h>
41#include <linux/devfreq.h>
42#include <linux/nls.h>
43#include <linux/of.h>
44#include "ufshcd.h"
45#include "ufs_quirks.h"
46#include "unipro.h"
47
48#define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
49                                 UTP_TASK_REQ_COMPL |\
50                                 UFSHCD_ERROR_MASK)
51/* UIC command timeout, unit: ms */
52#define UIC_CMD_TIMEOUT 500
53
54/* NOP OUT retries waiting for NOP IN response */
55#define NOP_OUT_RETRIES    10
56/* Timeout after 30 msecs if NOP OUT hangs without response */
57#define NOP_OUT_TIMEOUT    30 /* msecs */
58
59/* Query request retries */
60#define QUERY_REQ_RETRIES 10
61/* Query request timeout */
62#define QUERY_REQ_TIMEOUT 30 /* msec */
63/*
64 * Query request timeout for fDeviceInit flag
65 * fDeviceInit query response time for some devices is too large that default
66 * QUERY_REQ_TIMEOUT may not be enough for such devices.
67 */
68#define QUERY_FDEVICEINIT_REQ_TIMEOUT 600 /* msec */
69
70/* Task management command timeout */
71#define TM_CMD_TIMEOUT  100 /* msecs */
72
73/* maximum number of retries for a general UIC command  */
74#define UFS_UIC_COMMAND_RETRIES 3
75
76/* maximum number of link-startup retries */
77#define DME_LINKSTARTUP_RETRIES 3
78
79/* Maximum retries for Hibern8 enter */
80#define UIC_HIBERN8_ENTER_RETRIES 3
81
82/* maximum number of reset retries before giving up */
83#define MAX_HOST_RESET_RETRIES 5
84
85/* Expose the flag value from utp_upiu_query.value */
86#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
88/* Interrupt aggregation default timeout, unit: 40us */
89#define INT_AGGR_DEF_TO 0x02
90
91#define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
92        ({                                                              \
93                int _ret;                                               \
94                if (_on)                                                \
95                        _ret = ufshcd_enable_vreg(_dev, _vreg);         \
96                else                                                    \
97                        _ret = ufshcd_disable_vreg(_dev, _vreg);        \
98                _ret;                                                   \
99        })
100
101static u32 ufs_query_desc_max_size[] = {
102        QUERY_DESC_DEVICE_MAX_SIZE,
103        QUERY_DESC_CONFIGURAION_MAX_SIZE,
104        QUERY_DESC_UNIT_MAX_SIZE,
105        QUERY_DESC_RFU_MAX_SIZE,
106        QUERY_DESC_INTERCONNECT_MAX_SIZE,
107        QUERY_DESC_STRING_MAX_SIZE,
108        QUERY_DESC_RFU_MAX_SIZE,
109        QUERY_DESC_GEOMETRY_MAX_SIZE,
110        QUERY_DESC_POWER_MAX_SIZE,
111        QUERY_DESC_RFU_MAX_SIZE,
112};
113
114enum {
115        UFSHCD_MAX_CHANNEL      = 0,
116        UFSHCD_MAX_ID           = 1,
117        UFSHCD_CMD_PER_LUN      = 32,
118        UFSHCD_CAN_QUEUE        = 32,
119};
120
121/* UFSHCD states */
122enum {
123        UFSHCD_STATE_RESET,
124        UFSHCD_STATE_ERROR,
125        UFSHCD_STATE_OPERATIONAL,
126        UFSHCD_STATE_EH_SCHEDULED,
127};
128
129/* UFSHCD error handling flags */
130enum {
131        UFSHCD_EH_IN_PROGRESS = (1 << 0),
132};
133
134/* UFSHCD UIC layer error flags */
135enum {
136        UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
137        UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
138        UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
139        UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
140        UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
141        UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
142};
143
144/* Interrupt configuration options */
145enum {
146        UFSHCD_INT_DISABLE,
147        UFSHCD_INT_ENABLE,
148        UFSHCD_INT_CLEAR,
149};
150
151#define ufshcd_set_eh_in_progress(h) \
152        (h->eh_flags |= UFSHCD_EH_IN_PROGRESS)
153#define ufshcd_eh_in_progress(h) \
154        (h->eh_flags & UFSHCD_EH_IN_PROGRESS)
155#define ufshcd_clear_eh_in_progress(h) \
156        (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
157
158#define ufshcd_set_ufs_dev_active(h) \
159        ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
160#define ufshcd_set_ufs_dev_sleep(h) \
161        ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
162#define ufshcd_set_ufs_dev_poweroff(h) \
163        ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
164#define ufshcd_is_ufs_dev_active(h) \
165        ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
166#define ufshcd_is_ufs_dev_sleep(h) \
167        ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
168#define ufshcd_is_ufs_dev_poweroff(h) \
169        ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
170
171static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
172        {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
173        {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
174        {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
175        {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
176        {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
177        {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
178};
179
180static inline enum ufs_dev_pwr_mode
181ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
182{
183        return ufs_pm_lvl_states[lvl].dev_state;
184}
185
186static inline enum uic_link_state
187ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
188{
189        return ufs_pm_lvl_states[lvl].link_state;
190}
191
192static struct ufs_dev_fix ufs_fixups[] = {
193        /* UFS cards deviations table */
194        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
195                UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
196        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
197        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
198                UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
199        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
200                UFS_DEVICE_NO_FASTAUTO),
201        UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
202                UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
203        UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
204                UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
205        UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
206                UFS_DEVICE_QUIRK_PA_TACTIVATE),
207        UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
208                UFS_DEVICE_QUIRK_PA_TACTIVATE),
209        UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
210        UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
211                UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
212
213        END_FIX
214};
215
216static void ufshcd_tmc_handler(struct ufs_hba *hba);
217static void ufshcd_async_scan(void *data, async_cookie_t cookie);
218static int ufshcd_reset_and_restore(struct ufs_hba *hba);
219static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
220static void ufshcd_hba_exit(struct ufs_hba *hba);
221static int ufshcd_probe_hba(struct ufs_hba *hba);
222static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
223                                 bool skip_ref_clk);
224static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
225static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
226static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
227static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
228static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
229static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
230static irqreturn_t ufshcd_intr(int irq, void *__hba);
231static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
232                struct ufs_pa_layer_attr *desired_pwr_mode);
233static int ufshcd_change_power_mode(struct ufs_hba *hba,
234                             struct ufs_pa_layer_attr *pwr_mode);
235static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
236{
237        return tag >= 0 && tag < hba->nutrs;
238}
239
240static inline int ufshcd_enable_irq(struct ufs_hba *hba)
241{
242        int ret = 0;
243
244        if (!hba->is_irq_enabled) {
245                ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
246                                hba);
247                if (ret)
248                        dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
249                                __func__, ret);
250                hba->is_irq_enabled = true;
251        }
252
253        return ret;
254}
255
256static inline void ufshcd_disable_irq(struct ufs_hba *hba)
257{
258        if (hba->is_irq_enabled) {
259                free_irq(hba->irq, hba);
260                hba->is_irq_enabled = false;
261        }
262}
263
264/* replace non-printable or non-ASCII characters with spaces */
265static inline void ufshcd_remove_non_printable(char *val)
266{
267        if (!val)
268                return;
269
270        if (*val < 0x20 || *val > 0x7e)
271                *val = ' ';
272}
273
274/*
275 * ufshcd_wait_for_register - wait for register value to change
276 * @hba - per-adapter interface
277 * @reg - mmio register offset
278 * @mask - mask to apply to read register value
279 * @val - wait condition
280 * @interval_us - polling interval in microsecs
281 * @timeout_ms - timeout in millisecs
282 * @can_sleep - perform sleep or just spin
283 *
284 * Returns -ETIMEDOUT on error, zero on success
285 */
286int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
287                                u32 val, unsigned long interval_us,
288                                unsigned long timeout_ms, bool can_sleep)
289{
290        int err = 0;
291        unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
292
293        /* ignore bits that we don't intend to wait on */
294        val = val & mask;
295
296        while ((ufshcd_readl(hba, reg) & mask) != val) {
297                if (can_sleep)
298                        usleep_range(interval_us, interval_us + 50);
299                else
300                        udelay(interval_us);
301                if (time_after(jiffies, timeout)) {
302                        if ((ufshcd_readl(hba, reg) & mask) != val)
303                                err = -ETIMEDOUT;
304                        break;
305                }
306        }
307
308        return err;
309}
310
311/**
312 * ufshcd_get_intr_mask - Get the interrupt bit mask
313 * @hba - Pointer to adapter instance
314 *
315 * Returns interrupt bit mask per version
316 */
317static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
318{
319        if (hba->ufs_version == UFSHCI_VERSION_10)
320                return INTERRUPT_MASK_ALL_VER_10;
321        else
322                return INTERRUPT_MASK_ALL_VER_11;
323}
324
325/**
326 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
327 * @hba - Pointer to adapter instance
328 *
329 * Returns UFSHCI version supported by the controller
330 */
331static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
332{
333        if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
334                return ufshcd_vops_get_ufs_hci_version(hba);
335
336        return ufshcd_readl(hba, REG_UFS_VERSION);
337}
338
339/**
340 * ufshcd_is_device_present - Check if any device connected to
341 *                            the host controller
342 * @hba: pointer to adapter instance
343 *
344 * Returns 1 if device present, 0 if no device detected
345 */
346static inline int ufshcd_is_device_present(struct ufs_hba *hba)
347{
348        return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
349                                                DEVICE_PRESENT) ? 1 : 0;
350}
351
352/**
353 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
354 * @lrb: pointer to local command reference block
355 *
356 * This function is used to get the OCS field from UTRD
357 * Returns the OCS field in the UTRD
358 */
359static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
360{
361        return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
362}
363
364/**
365 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
366 * @task_req_descp: pointer to utp_task_req_desc structure
367 *
368 * This function is used to get the OCS field from UTMRD
369 * Returns the OCS field in the UTMRD
370 */
371static inline int
372ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
373{
374        return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
375}
376
377/**
378 * ufshcd_get_tm_free_slot - get a free slot for task management request
379 * @hba: per adapter instance
380 * @free_slot: pointer to variable with available slot value
381 *
382 * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
383 * Returns 0 if free slot is not available, else return 1 with tag value
384 * in @free_slot.
385 */
386static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
387{
388        int tag;
389        bool ret = false;
390
391        if (!free_slot)
392                goto out;
393
394        do {
395                tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
396                if (tag >= hba->nutmrs)
397                        goto out;
398        } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
399
400        *free_slot = tag;
401        ret = true;
402out:
403        return ret;
404}
405
406static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
407{
408        clear_bit_unlock(slot, &hba->tm_slots_in_use);
409}
410
411/**
412 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
413 * @hba: per adapter instance
414 * @pos: position of the bit to be cleared
415 */
416static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
417{
418        ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
419}
420
421/**
422 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
423 * @hba: per adapter instance
424 * @tag: position of the bit to be cleared
425 */
426static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
427{
428        __clear_bit(tag, &hba->outstanding_reqs);
429}
430
431/**
432 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
433 * @reg: Register value of host controller status
434 *
435 * Returns integer, 0 on Success and positive value if failed
436 */
437static inline int ufshcd_get_lists_status(u32 reg)
438{
439        /*
440         * The mask 0xFF is for the following HCS register bits
441         * Bit          Description
442         *  0           Device Present
443         *  1           UTRLRDY
444         *  2           UTMRLRDY
445         *  3           UCRDY
446         * 4-7          reserved
447         */
448        return ((reg & 0xFF) >> 1) ^ 0x07;
449}
450
451/**
452 * ufshcd_get_uic_cmd_result - Get the UIC command result
453 * @hba: Pointer to adapter instance
454 *
455 * This function gets the result of UIC command completion
456 * Returns 0 on success, non zero value on error
457 */
458static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
459{
460        return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
461               MASK_UIC_COMMAND_RESULT;
462}
463
464/**
465 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
466 * @hba: Pointer to adapter instance
467 *
468 * This function gets UIC command argument3
469 * Returns 0 on success, non zero value on error
470 */
471static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
472{
473        return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
474}
475
476/**
477 * ufshcd_get_req_rsp - returns the TR response transaction type
478 * @ucd_rsp_ptr: pointer to response UPIU
479 */
480static inline int
481ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
482{
483        return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
484}
485
486/**
487 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
488 * @ucd_rsp_ptr: pointer to response UPIU
489 *
490 * This function gets the response status and scsi_status from response UPIU
491 * Returns the response result code.
492 */
493static inline int
494ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
495{
496        return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
497}
498
499/*
500 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
501 *                              from response UPIU
502 * @ucd_rsp_ptr: pointer to response UPIU
503 *
504 * Return the data segment length.
505 */
506static inline unsigned int
507ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
508{
509        return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
510                MASK_RSP_UPIU_DATA_SEG_LEN;
511}
512
513/**
514 * ufshcd_is_exception_event - Check if the device raised an exception event
515 * @ucd_rsp_ptr: pointer to response UPIU
516 *
517 * The function checks if the device raised an exception event indicated in
518 * the Device Information field of response UPIU.
519 *
520 * Returns true if exception is raised, false otherwise.
521 */
522static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
523{
524        return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
525                        MASK_RSP_EXCEPTION_EVENT ? true : false;
526}
527
528/**
529 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
530 * @hba: per adapter instance
531 */
532static inline void
533ufshcd_reset_intr_aggr(struct ufs_hba *hba)
534{
535        ufshcd_writel(hba, INT_AGGR_ENABLE |
536                      INT_AGGR_COUNTER_AND_TIMER_RESET,
537                      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
538}
539
540/**
541 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
542 * @hba: per adapter instance
543 * @cnt: Interrupt aggregation counter threshold
544 * @tmout: Interrupt aggregation timeout value
545 */
546static inline void
547ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
548{
549        ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
550                      INT_AGGR_COUNTER_THLD_VAL(cnt) |
551                      INT_AGGR_TIMEOUT_VAL(tmout),
552                      REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
553}
554
555/**
556 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
557 * @hba: per adapter instance
558 */
559static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
560{
561        ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
562}
563
564/**
565 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
566 *                      When run-stop registers are set to 1, it indicates the
567 *                      host controller that it can process the requests
568 * @hba: per adapter instance
569 */
570static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
571{
572        ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
573                      REG_UTP_TASK_REQ_LIST_RUN_STOP);
574        ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
575                      REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
576}
577
578/**
579 * ufshcd_hba_start - Start controller initialization sequence
580 * @hba: per adapter instance
581 */
582static inline void ufshcd_hba_start(struct ufs_hba *hba)
583{
584        ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
585}
586
587/**
588 * ufshcd_is_hba_active - Get controller state
589 * @hba: per adapter instance
590 *
591 * Returns zero if controller is active, 1 otherwise
592 */
593static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
594{
595        return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
596}
597
598u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
599{
600        /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
601        if ((hba->ufs_version == UFSHCI_VERSION_10) ||
602            (hba->ufs_version == UFSHCI_VERSION_11))
603                return UFS_UNIPRO_VER_1_41;
604        else
605                return UFS_UNIPRO_VER_1_6;
606}
607EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
608
609static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
610{
611        /*
612         * If both host and device support UniPro ver1.6 or later, PA layer
613         * parameters tuning happens during link startup itself.
614         *
615         * We can manually tune PA layer parameters if either host or device
616         * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
617         * logic simple, we will only do manual tuning if local unipro version
618         * doesn't support ver1.6 or later.
619         */
620        if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
621                return true;
622        else
623                return false;
624}
625
626static void ufshcd_ungate_work(struct work_struct *work)
627{
628        int ret;
629        unsigned long flags;
630        struct ufs_hba *hba = container_of(work, struct ufs_hba,
631                        clk_gating.ungate_work);
632
633        cancel_delayed_work_sync(&hba->clk_gating.gate_work);
634
635        spin_lock_irqsave(hba->host->host_lock, flags);
636        if (hba->clk_gating.state == CLKS_ON) {
637                spin_unlock_irqrestore(hba->host->host_lock, flags);
638                goto unblock_reqs;
639        }
640
641        spin_unlock_irqrestore(hba->host->host_lock, flags);
642        ufshcd_setup_clocks(hba, true);
643
644        /* Exit from hibern8 */
645        if (ufshcd_can_hibern8_during_gating(hba)) {
646                /* Prevent gating in this path */
647                hba->clk_gating.is_suspended = true;
648                if (ufshcd_is_link_hibern8(hba)) {
649                        ret = ufshcd_uic_hibern8_exit(hba);
650                        if (ret)
651                                dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
652                                        __func__, ret);
653                        else
654                                ufshcd_set_link_active(hba);
655                }
656                hba->clk_gating.is_suspended = false;
657        }
658unblock_reqs:
659        if (ufshcd_is_clkscaling_enabled(hba))
660                devfreq_resume_device(hba->devfreq);
661        scsi_unblock_requests(hba->host);
662}
663
664/**
665 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
666 * Also, exit from hibern8 mode and set the link as active.
667 * @hba: per adapter instance
668 * @async: This indicates whether caller should ungate clocks asynchronously.
669 */
670int ufshcd_hold(struct ufs_hba *hba, bool async)
671{
672        int rc = 0;
673        unsigned long flags;
674
675        if (!ufshcd_is_clkgating_allowed(hba))
676                goto out;
677        spin_lock_irqsave(hba->host->host_lock, flags);
678        hba->clk_gating.active_reqs++;
679
680        if (ufshcd_eh_in_progress(hba)) {
681                spin_unlock_irqrestore(hba->host->host_lock, flags);
682                return 0;
683        }
684
685start:
686        switch (hba->clk_gating.state) {
687        case CLKS_ON:
688                break;
689        case REQ_CLKS_OFF:
690                if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
691                        hba->clk_gating.state = CLKS_ON;
692                        break;
693                }
694                /*
695                 * If we here, it means gating work is either done or
696                 * currently running. Hence, fall through to cancel gating
697                 * work and to enable clocks.
698                 */
699        case CLKS_OFF:
700                scsi_block_requests(hba->host);
701                hba->clk_gating.state = REQ_CLKS_ON;
702                schedule_work(&hba->clk_gating.ungate_work);
703                /*
704                 * fall through to check if we should wait for this
705                 * work to be done or not.
706                 */
707        case REQ_CLKS_ON:
708                if (async) {
709                        rc = -EAGAIN;
710                        hba->clk_gating.active_reqs--;
711                        break;
712                }
713
714                spin_unlock_irqrestore(hba->host->host_lock, flags);
715                flush_work(&hba->clk_gating.ungate_work);
716                /* Make sure state is CLKS_ON before returning */
717                spin_lock_irqsave(hba->host->host_lock, flags);
718                goto start;
719        default:
720                dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
721                                __func__, hba->clk_gating.state);
722                break;
723        }
724        spin_unlock_irqrestore(hba->host->host_lock, flags);
725out:
726        return rc;
727}
728EXPORT_SYMBOL_GPL(ufshcd_hold);
729
730static void ufshcd_gate_work(struct work_struct *work)
731{
732        struct ufs_hba *hba = container_of(work, struct ufs_hba,
733                        clk_gating.gate_work.work);
734        unsigned long flags;
735
736        spin_lock_irqsave(hba->host->host_lock, flags);
737        if (hba->clk_gating.is_suspended) {
738                hba->clk_gating.state = CLKS_ON;
739                goto rel_lock;
740        }
741
742        if (hba->clk_gating.active_reqs
743                || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
744                || hba->lrb_in_use || hba->outstanding_tasks
745                || hba->active_uic_cmd || hba->uic_async_done)
746                goto rel_lock;
747
748        spin_unlock_irqrestore(hba->host->host_lock, flags);
749
750        /* put the link into hibern8 mode before turning off clocks */
751        if (ufshcd_can_hibern8_during_gating(hba)) {
752                if (ufshcd_uic_hibern8_enter(hba)) {
753                        hba->clk_gating.state = CLKS_ON;
754                        goto out;
755                }
756                ufshcd_set_link_hibern8(hba);
757        }
758
759        if (ufshcd_is_clkscaling_enabled(hba)) {
760                devfreq_suspend_device(hba->devfreq);
761                hba->clk_scaling.window_start_t = 0;
762        }
763
764        if (!ufshcd_is_link_active(hba))
765                ufshcd_setup_clocks(hba, false);
766        else
767                /* If link is active, device ref_clk can't be switched off */
768                __ufshcd_setup_clocks(hba, false, true);
769
770        /*
771         * In case you are here to cancel this work the gating state
772         * would be marked as REQ_CLKS_ON. In this case keep the state
773         * as REQ_CLKS_ON which would anyway imply that clocks are off
774         * and a request to turn them on is pending. By doing this way,
775         * we keep the state machine in tact and this would ultimately
776         * prevent from doing cancel work multiple times when there are
777         * new requests arriving before the current cancel work is done.
778         */
779        spin_lock_irqsave(hba->host->host_lock, flags);
780        if (hba->clk_gating.state == REQ_CLKS_OFF)
781                hba->clk_gating.state = CLKS_OFF;
782
783rel_lock:
784        spin_unlock_irqrestore(hba->host->host_lock, flags);
785out:
786        return;
787}
788
789/* host lock must be held before calling this variant */
790static void __ufshcd_release(struct ufs_hba *hba)
791{
792        if (!ufshcd_is_clkgating_allowed(hba))
793                return;
794
795        hba->clk_gating.active_reqs--;
796
797        if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
798                || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
799                || hba->lrb_in_use || hba->outstanding_tasks
800                || hba->active_uic_cmd || hba->uic_async_done
801                || ufshcd_eh_in_progress(hba))
802                return;
803
804        hba->clk_gating.state = REQ_CLKS_OFF;
805        schedule_delayed_work(&hba->clk_gating.gate_work,
806                        msecs_to_jiffies(hba->clk_gating.delay_ms));
807}
808
809void ufshcd_release(struct ufs_hba *hba)
810{
811        unsigned long flags;
812
813        spin_lock_irqsave(hba->host->host_lock, flags);
814        __ufshcd_release(hba);
815        spin_unlock_irqrestore(hba->host->host_lock, flags);
816}
817EXPORT_SYMBOL_GPL(ufshcd_release);
818
819static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
820                struct device_attribute *attr, char *buf)
821{
822        struct ufs_hba *hba = dev_get_drvdata(dev);
823
824        return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
825}
826
827static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
828                struct device_attribute *attr, const char *buf, size_t count)
829{
830        struct ufs_hba *hba = dev_get_drvdata(dev);
831        unsigned long flags, value;
832
833        if (kstrtoul(buf, 0, &value))
834                return -EINVAL;
835
836        spin_lock_irqsave(hba->host->host_lock, flags);
837        hba->clk_gating.delay_ms = value;
838        spin_unlock_irqrestore(hba->host->host_lock, flags);
839        return count;
840}
841
842static void ufshcd_init_clk_gating(struct ufs_hba *hba)
843{
844        if (!ufshcd_is_clkgating_allowed(hba))
845                return;
846
847        hba->clk_gating.delay_ms = 150;
848        INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
849        INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
850
851        hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
852        hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
853        sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
854        hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
855        hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR;
856        if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
857                dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
858}
859
860static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
861{
862        if (!ufshcd_is_clkgating_allowed(hba))
863                return;
864        device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
865        cancel_work_sync(&hba->clk_gating.ungate_work);
866        cancel_delayed_work_sync(&hba->clk_gating.gate_work);
867}
868
869/* Must be called with host lock acquired */
870static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
871{
872        if (!ufshcd_is_clkscaling_enabled(hba))
873                return;
874
875        if (!hba->clk_scaling.is_busy_started) {
876                hba->clk_scaling.busy_start_t = ktime_get();
877                hba->clk_scaling.is_busy_started = true;
878        }
879}
880
881static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
882{
883        struct ufs_clk_scaling *scaling = &hba->clk_scaling;
884
885        if (!ufshcd_is_clkscaling_enabled(hba))
886                return;
887
888        if (!hba->outstanding_reqs && scaling->is_busy_started) {
889                scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
890                                        scaling->busy_start_t));
891                scaling->busy_start_t = ktime_set(0, 0);
892                scaling->is_busy_started = false;
893        }
894}
895/**
896 * ufshcd_send_command - Send SCSI or device management commands
897 * @hba: per adapter instance
898 * @task_tag: Task tag of the command
899 */
900static inline
901void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
902{
903        ufshcd_clk_scaling_start_busy(hba);
904        __set_bit(task_tag, &hba->outstanding_reqs);
905        ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
906}
907
908/**
909 * ufshcd_copy_sense_data - Copy sense data in case of check condition
910 * @lrb - pointer to local reference block
911 */
912static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
913{
914        int len;
915        if (lrbp->sense_buffer &&
916            ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
917                len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
918                memcpy(lrbp->sense_buffer,
919                        lrbp->ucd_rsp_ptr->sr.sense_data,
920                        min_t(int, len, SCSI_SENSE_BUFFERSIZE));
921        }
922}
923
924/**
925 * ufshcd_copy_query_response() - Copy the Query Response and the data
926 * descriptor
927 * @hba: per adapter instance
928 * @lrb - pointer to local reference block
929 */
930static
931int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
932{
933        struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
934
935        memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
936
937        /* Get the descriptor */
938        if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
939                u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
940                                GENERAL_UPIU_REQUEST_SIZE;
941                u16 resp_len;
942                u16 buf_len;
943
944                /* data segment length */
945                resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
946                                                MASK_QUERY_DATA_SEG_LEN;
947                buf_len = be16_to_cpu(
948                                hba->dev_cmd.query.request.upiu_req.length);
949                if (likely(buf_len >= resp_len)) {
950                        memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
951                } else {
952                        dev_warn(hba->dev,
953                                "%s: Response size is bigger than buffer",
954                                __func__);
955                        return -EINVAL;
956                }
957        }
958
959        return 0;
960}
961
962/**
963 * ufshcd_hba_capabilities - Read controller capabilities
964 * @hba: per adapter instance
965 */
966static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
967{
968        hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
969
970        /* nutrs and nutmrs are 0 based values */
971        hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
972        hba->nutmrs =
973        ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
974}
975
976/**
977 * ufshcd_ready_for_uic_cmd - Check if controller is ready
978 *                            to accept UIC commands
979 * @hba: per adapter instance
980 * Return true on success, else false
981 */
982static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
983{
984        if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
985                return true;
986        else
987                return false;
988}
989
990/**
991 * ufshcd_get_upmcrs - Get the power mode change request status
992 * @hba: Pointer to adapter instance
993 *
994 * This function gets the UPMCRS field of HCS register
995 * Returns value of UPMCRS field
996 */
997static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
998{
999        return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1000}
1001
1002/**
1003 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1004 * @hba: per adapter instance
1005 * @uic_cmd: UIC command
1006 *
1007 * Mutex must be held.
1008 */
1009static inline void
1010ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1011{
1012        WARN_ON(hba->active_uic_cmd);
1013
1014        hba->active_uic_cmd = uic_cmd;
1015
1016        /* Write Args */
1017        ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1018        ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1019        ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
1020
1021        /* Write UIC Cmd */
1022        ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
1023                      REG_UIC_COMMAND);
1024}
1025
1026/**
1027 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
1028 * @hba: per adapter instance
1029 * @uic_command: UIC command
1030 *
1031 * Must be called with mutex held.
1032 * Returns 0 only if success.
1033 */
1034static int
1035ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1036{
1037        int ret;
1038        unsigned long flags;
1039
1040        if (wait_for_completion_timeout(&uic_cmd->done,
1041                                        msecs_to_jiffies(UIC_CMD_TIMEOUT)))
1042                ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
1043        else
1044                ret = -ETIMEDOUT;
1045
1046        spin_lock_irqsave(hba->host->host_lock, flags);
1047        hba->active_uic_cmd = NULL;
1048        spin_unlock_irqrestore(hba->host->host_lock, flags);
1049
1050        return ret;
1051}
1052
1053/**
1054 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1055 * @hba: per adapter instance
1056 * @uic_cmd: UIC command
1057 * @completion: initialize the completion only if this is set to true
1058 *
1059 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
1060 * with mutex held and host_lock locked.
1061 * Returns 0 only if success.
1062 */
1063static int
1064__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
1065                      bool completion)
1066{
1067        if (!ufshcd_ready_for_uic_cmd(hba)) {
1068                dev_err(hba->dev,
1069                        "Controller not ready to accept UIC commands\n");
1070                return -EIO;
1071        }
1072
1073        if (completion)
1074                init_completion(&uic_cmd->done);
1075
1076        ufshcd_dispatch_uic_cmd(hba, uic_cmd);
1077
1078        return 0;
1079}
1080
1081/**
1082 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1083 * @hba: per adapter instance
1084 * @uic_cmd: UIC command
1085 *
1086 * Returns 0 only if success.
1087 */
1088static int
1089ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1090{
1091        int ret;
1092        unsigned long flags;
1093
1094        ufshcd_hold(hba, false);
1095        mutex_lock(&hba->uic_cmd_mutex);
1096        ufshcd_add_delay_before_dme_cmd(hba);
1097
1098        spin_lock_irqsave(hba->host->host_lock, flags);
1099        ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
1100        spin_unlock_irqrestore(hba->host->host_lock, flags);
1101        if (!ret)
1102                ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
1103
1104        mutex_unlock(&hba->uic_cmd_mutex);
1105
1106        ufshcd_release(hba);
1107        return ret;
1108}
1109
1110/**
1111 * ufshcd_map_sg - Map scatter-gather list to prdt
1112 * @lrbp - pointer to local reference block
1113 *
1114 * Returns 0 in case of success, non-zero value in case of failure
1115 */
1116static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1117{
1118        struct ufshcd_sg_entry *prd_table;
1119        struct scatterlist *sg;
1120        struct scsi_cmnd *cmd;
1121        int sg_segments;
1122        int i;
1123
1124        cmd = lrbp->cmd;
1125        sg_segments = scsi_dma_map(cmd);
1126        if (sg_segments < 0)
1127                return sg_segments;
1128
1129        if (sg_segments) {
1130                if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
1131                        lrbp->utr_descriptor_ptr->prd_table_length =
1132                                cpu_to_le16((u16)(sg_segments *
1133                                        sizeof(struct ufshcd_sg_entry)));
1134                else
1135                        lrbp->utr_descriptor_ptr->prd_table_length =
1136                                cpu_to_le16((u16) (sg_segments));
1137
1138                prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
1139
1140                scsi_for_each_sg(cmd, sg, sg_segments, i) {
1141                        prd_table[i].size  =
1142                                cpu_to_le32(((u32) sg_dma_len(sg))-1);
1143                        prd_table[i].base_addr =
1144                                cpu_to_le32(lower_32_bits(sg->dma_address));
1145                        prd_table[i].upper_addr =
1146                                cpu_to_le32(upper_32_bits(sg->dma_address));
1147                        prd_table[i].reserved = 0;
1148                }
1149        } else {
1150                lrbp->utr_descriptor_ptr->prd_table_length = 0;
1151        }
1152
1153        return 0;
1154}
1155
1156/**
1157 * ufshcd_enable_intr - enable interrupts
1158 * @hba: per adapter instance
1159 * @intrs: interrupt bits
1160 */
1161static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
1162{
1163        u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1164
1165        if (hba->ufs_version == UFSHCI_VERSION_10) {
1166                u32 rw;
1167                rw = set & INTERRUPT_MASK_RW_VER_10;
1168                set = rw | ((set ^ intrs) & intrs);
1169        } else {
1170                set |= intrs;
1171        }
1172
1173        ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1174}
1175
1176/**
1177 * ufshcd_disable_intr - disable interrupts
1178 * @hba: per adapter instance
1179 * @intrs: interrupt bits
1180 */
1181static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
1182{
1183        u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1184
1185        if (hba->ufs_version == UFSHCI_VERSION_10) {
1186                u32 rw;
1187                rw = (set & INTERRUPT_MASK_RW_VER_10) &
1188                        ~(intrs & INTERRUPT_MASK_RW_VER_10);
1189                set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
1190
1191        } else {
1192                set &= ~intrs;
1193        }
1194
1195        ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1196}
1197
1198/**
1199 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
1200 * descriptor according to request
1201 * @lrbp: pointer to local reference block
1202 * @upiu_flags: flags required in the header
1203 * @cmd_dir: requests data direction
1204 */
1205static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
1206                        u32 *upiu_flags, enum dma_data_direction cmd_dir)
1207{
1208        struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
1209        u32 data_direction;
1210        u32 dword_0;
1211
1212        if (cmd_dir == DMA_FROM_DEVICE) {
1213                data_direction = UTP_DEVICE_TO_HOST;
1214                *upiu_flags = UPIU_CMD_FLAGS_READ;
1215        } else if (cmd_dir == DMA_TO_DEVICE) {
1216                data_direction = UTP_HOST_TO_DEVICE;
1217                *upiu_flags = UPIU_CMD_FLAGS_WRITE;
1218        } else {
1219                data_direction = UTP_NO_DATA_TRANSFER;
1220                *upiu_flags = UPIU_CMD_FLAGS_NONE;
1221        }
1222
1223        dword_0 = data_direction | (lrbp->command_type
1224                                << UPIU_COMMAND_TYPE_OFFSET);
1225        if (lrbp->intr_cmd)
1226                dword_0 |= UTP_REQ_DESC_INT_CMD;
1227
1228        /* Transfer request descriptor header fields */
1229        req_desc->header.dword_0 = cpu_to_le32(dword_0);
1230        /* dword_1 is reserved, hence it is set to 0 */
1231        req_desc->header.dword_1 = 0;
1232        /*
1233         * assigning invalid value for command status. Controller
1234         * updates OCS on command completion, with the command
1235         * status
1236         */
1237        req_desc->header.dword_2 =
1238                cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
1239        /* dword_3 is reserved, hence it is set to 0 */
1240        req_desc->header.dword_3 = 0;
1241
1242        req_desc->prd_table_length = 0;
1243}
1244
1245/**
1246 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
1247 * for scsi commands
1248 * @lrbp - local reference block pointer
1249 * @upiu_flags - flags
1250 */
1251static
1252void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
1253{
1254        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1255        unsigned short cdb_len;
1256
1257        /* command descriptor fields */
1258        ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1259                                UPIU_TRANSACTION_COMMAND, upiu_flags,
1260                                lrbp->lun, lrbp->task_tag);
1261        ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1262                                UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
1263
1264        /* Total EHS length and Data segment length will be zero */
1265        ucd_req_ptr->header.dword_2 = 0;
1266
1267        ucd_req_ptr->sc.exp_data_transfer_len =
1268                cpu_to_be32(lrbp->cmd->sdb.length);
1269
1270        cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
1271        memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
1272        memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
1273
1274        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1275}
1276
1277/**
1278 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
1279 * for query requsts
1280 * @hba: UFS hba
1281 * @lrbp: local reference block pointer
1282 * @upiu_flags: flags
1283 */
1284static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
1285                                struct ufshcd_lrb *lrbp, u32 upiu_flags)
1286{
1287        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1288        struct ufs_query *query = &hba->dev_cmd.query;
1289        u16 len = be16_to_cpu(query->request.upiu_req.length);
1290        u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
1291
1292        /* Query request header */
1293        ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1294                        UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
1295                        lrbp->lun, lrbp->task_tag);
1296        ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1297                        0, query->request.query_func, 0, 0);
1298
1299        /* Data segment length only need for WRITE_DESC */
1300        if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
1301                ucd_req_ptr->header.dword_2 =
1302                        UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
1303        else
1304                ucd_req_ptr->header.dword_2 = 0;
1305
1306        /* Copy the Query Request buffer as is */
1307        memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
1308                        QUERY_OSF_SIZE);
1309
1310        /* Copy the Descriptor */
1311        if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
1312                memcpy(descp, query->descriptor, len);
1313
1314        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1315}
1316
1317static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
1318{
1319        struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1320
1321        memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
1322
1323        /* command descriptor fields */
1324        ucd_req_ptr->header.dword_0 =
1325                UPIU_HEADER_DWORD(
1326                        UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
1327        /* clear rest of the fields of basic header */
1328        ucd_req_ptr->header.dword_1 = 0;
1329        ucd_req_ptr->header.dword_2 = 0;
1330
1331        memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1332}
1333
1334/**
1335 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
1336 *                           for Device Management Purposes
1337 * @hba - per adapter instance
1338 * @lrb - pointer to local reference block
1339 */
1340static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1341{
1342        u32 upiu_flags;
1343        int ret = 0;
1344
1345        if (hba->ufs_version == UFSHCI_VERSION_20)
1346                lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
1347        else
1348                lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
1349
1350        ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
1351        if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
1352                ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
1353        else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
1354                ufshcd_prepare_utp_nop_upiu(lrbp);
1355        else
1356                ret = -EINVAL;
1357
1358        return ret;
1359}
1360
1361/**
1362 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
1363 *                         for SCSI Purposes
1364 * @hba - per adapter instance
1365 * @lrb - pointer to local reference block
1366 */
1367static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1368{
1369        u32 upiu_flags;
1370        int ret = 0;
1371
1372        if (hba->ufs_version == UFSHCI_VERSION_20)
1373                lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
1374        else
1375                lrbp->command_type = UTP_CMD_TYPE_SCSI;
1376
1377        if (likely(lrbp->cmd)) {
1378                ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
1379                                                lrbp->cmd->sc_data_direction);
1380                ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
1381        } else {
1382                ret = -EINVAL;
1383        }
1384
1385        return ret;
1386}
1387
1388/*
1389 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN
1390 * @scsi_lun: scsi LUN id
1391 *
1392 * Returns UPIU LUN id
1393 */
1394static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
1395{
1396        if (scsi_is_wlun(scsi_lun))
1397                return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
1398                        | UFS_UPIU_WLUN_ID;
1399        else
1400                return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
1401}
1402
1403/**
1404 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
1405 * @scsi_lun: UPIU W-LUN id
1406 *
1407 * Returns SCSI W-LUN id
1408 */
1409static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
1410{
1411        return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
1412}
1413
1414/**
1415 * ufshcd_queuecommand - main entry point for SCSI requests
1416 * @cmd: command from SCSI Midlayer
1417 * @done: call back function
1418 *
1419 * Returns 0 for success, non-zero in case of failure
1420 */
1421static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
1422{
1423        struct ufshcd_lrb *lrbp;
1424        struct ufs_hba *hba;
1425        unsigned long flags;
1426        int tag;
1427        int err = 0;
1428
1429        hba = shost_priv(host);
1430
1431        tag = cmd->request->tag;
1432        if (!ufshcd_valid_tag(hba, tag)) {
1433                dev_err(hba->dev,
1434                        "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
1435                        __func__, tag, cmd, cmd->request);
1436                BUG();
1437        }
1438
1439        spin_lock_irqsave(hba->host->host_lock, flags);
1440        switch (hba->ufshcd_state) {
1441        case UFSHCD_STATE_OPERATIONAL:
1442                break;
1443        case UFSHCD_STATE_EH_SCHEDULED:
1444        case UFSHCD_STATE_RESET:
1445                err = SCSI_MLQUEUE_HOST_BUSY;
1446                goto out_unlock;
1447        case UFSHCD_STATE_ERROR:
1448                set_host_byte(cmd, DID_ERROR);
1449                cmd->scsi_done(cmd);
1450                goto out_unlock;
1451        default:
1452                dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
1453                                __func__, hba->ufshcd_state);
1454                set_host_byte(cmd, DID_BAD_TARGET);
1455                cmd->scsi_done(cmd);
1456                goto out_unlock;
1457        }
1458
1459        /* if error handling is in progress, don't issue commands */
1460        if (ufshcd_eh_in_progress(hba)) {
1461                set_host_byte(cmd, DID_ERROR);
1462                cmd->scsi_done(cmd);
1463                goto out_unlock;
1464        }
1465        spin_unlock_irqrestore(hba->host->host_lock, flags);
1466
1467        /* acquire the tag to make sure device cmds don't use it */
1468        if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
1469                /*
1470                 * Dev manage command in progress, requeue the command.
1471                 * Requeuing the command helps in cases where the request *may*
1472                 * find different tag instead of waiting for dev manage command
1473                 * completion.
1474                 */
1475                err = SCSI_MLQUEUE_HOST_BUSY;
1476                goto out;
1477        }
1478
1479        err = ufshcd_hold(hba, true);
1480        if (err) {
1481                err = SCSI_MLQUEUE_HOST_BUSY;
1482                clear_bit_unlock(tag, &hba->lrb_in_use);
1483                goto out;
1484        }
1485        WARN_ON(hba->clk_gating.state != CLKS_ON);
1486
1487        lrbp = &hba->lrb[tag];
1488
1489        WARN_ON(lrbp->cmd);
1490        lrbp->cmd = cmd;
1491        lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
1492        lrbp->sense_buffer = cmd->sense_buffer;
1493        lrbp->task_tag = tag;
1494        lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
1495        lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
1496
1497        ufshcd_comp_scsi_upiu(hba, lrbp);
1498
1499        err = ufshcd_map_sg(hba, lrbp);
1500        if (err) {
1501                lrbp->cmd = NULL;
1502                clear_bit_unlock(tag, &hba->lrb_in_use);
1503                goto out;
1504        }
1505
1506        /* issue command to the controller */
1507        spin_lock_irqsave(hba->host->host_lock, flags);
1508        ufshcd_send_command(hba, tag);
1509out_unlock:
1510        spin_unlock_irqrestore(hba->host->host_lock, flags);
1511out:
1512        return err;
1513}
1514
1515static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
1516                struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
1517{
1518        lrbp->cmd = NULL;
1519        lrbp->sense_bufflen = 0;
1520        lrbp->sense_buffer = NULL;
1521        lrbp->task_tag = tag;
1522        lrbp->lun = 0; /* device management cmd is not specific to any LUN */
1523        lrbp->intr_cmd = true; /* No interrupt aggregation */
1524        hba->dev_cmd.type = cmd_type;
1525
1526        return ufshcd_comp_devman_upiu(hba, lrbp);
1527}
1528
1529static int
1530ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
1531{
1532        int err = 0;
1533        unsigned long flags;
1534        u32 mask = 1 << tag;
1535
1536        /* clear outstanding transaction before retry */
1537        spin_lock_irqsave(hba->host->host_lock, flags);
1538        ufshcd_utrl_clear(hba, tag);
1539        spin_unlock_irqrestore(hba->host->host_lock, flags);
1540
1541        /*
1542         * wait for for h/w to clear corresponding bit in door-bell.
1543         * max. wait is 1 sec.
1544         */
1545        err = ufshcd_wait_for_register(hba,
1546                        REG_UTP_TRANSFER_REQ_DOOR_BELL,
1547                        mask, ~mask, 1000, 1000, true);
1548
1549        return err;
1550}
1551
1552static int
1553ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1554{
1555        struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1556
1557        /* Get the UPIU response */
1558        query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
1559                                UPIU_RSP_CODE_OFFSET;
1560        return query_res->response;
1561}
1562
1563/**
1564 * ufshcd_dev_cmd_completion() - handles device management command responses
1565 * @hba: per adapter instance
1566 * @lrbp: pointer to local reference block
1567 */
1568static int
1569ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1570{
1571        int resp;
1572        int err = 0;
1573
1574        resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
1575
1576        switch (resp) {
1577        case UPIU_TRANSACTION_NOP_IN:
1578                if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
1579                        err = -EINVAL;
1580                        dev_err(hba->dev, "%s: unexpected response %x\n",
1581                                        __func__, resp);
1582                }
1583                break;
1584        case UPIU_TRANSACTION_QUERY_RSP:
1585                err = ufshcd_check_query_response(hba, lrbp);
1586                if (!err)
1587                        err = ufshcd_copy_query_response(hba, lrbp);
1588                break;
1589        case UPIU_TRANSACTION_REJECT_UPIU:
1590                /* TODO: handle Reject UPIU Response */
1591                err = -EPERM;
1592                dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
1593                                __func__);
1594                break;
1595        default:
1596                err = -EINVAL;
1597                dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
1598                                __func__, resp);
1599                break;
1600        }
1601
1602        return err;
1603}
1604
1605static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
1606                struct ufshcd_lrb *lrbp, int max_timeout)
1607{
1608        int err = 0;
1609        unsigned long time_left;
1610        unsigned long flags;
1611
1612        time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
1613                        msecs_to_jiffies(max_timeout));
1614
1615        spin_lock_irqsave(hba->host->host_lock, flags);
1616        hba->dev_cmd.complete = NULL;
1617        if (likely(time_left)) {
1618                err = ufshcd_get_tr_ocs(lrbp);
1619                if (!err)
1620                        err = ufshcd_dev_cmd_completion(hba, lrbp);
1621        }
1622        spin_unlock_irqrestore(hba->host->host_lock, flags);
1623
1624        if (!time_left) {
1625                err = -ETIMEDOUT;
1626                dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
1627                        __func__, lrbp->task_tag);
1628                if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
1629                        /* successfully cleared the command, retry if needed */
1630                        err = -EAGAIN;
1631                /*
1632                 * in case of an error, after clearing the doorbell,
1633                 * we also need to clear the outstanding_request
1634                 * field in hba
1635                 */
1636                ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
1637        }
1638
1639        return err;
1640}
1641
1642/**
1643 * ufshcd_get_dev_cmd_tag - Get device management command tag
1644 * @hba: per-adapter instance
1645 * @tag: pointer to variable with available slot value
1646 *
1647 * Get a free slot and lock it until device management command
1648 * completes.
1649 *
1650 * Returns false if free slot is unavailable for locking, else
1651 * return true with tag value in @tag.
1652 */
1653static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
1654{
1655        int tag;
1656        bool ret = false;
1657        unsigned long tmp;
1658
1659        if (!tag_out)
1660                goto out;
1661
1662        do {
1663                tmp = ~hba->lrb_in_use;
1664                tag = find_last_bit(&tmp, hba->nutrs);
1665                if (tag >= hba->nutrs)
1666                        goto out;
1667        } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
1668
1669        *tag_out = tag;
1670        ret = true;
1671out:
1672        return ret;
1673}
1674
1675static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
1676{
1677        clear_bit_unlock(tag, &hba->lrb_in_use);
1678}
1679
1680/**
1681 * ufshcd_exec_dev_cmd - API for sending device management requests
1682 * @hba - UFS hba
1683 * @cmd_type - specifies the type (NOP, Query...)
1684 * @timeout - time in seconds
1685 *
1686 * NOTE: Since there is only one available tag for device management commands,
1687 * it is expected you hold the hba->dev_cmd.lock mutex.
1688 */
1689static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
1690                enum dev_cmd_type cmd_type, int timeout)
1691{
1692        struct ufshcd_lrb *lrbp;
1693        int err;
1694        int tag;
1695        struct completion wait;
1696        unsigned long flags;
1697
1698        /*
1699         * Get free slot, sleep if slots are unavailable.
1700         * Even though we use wait_event() which sleeps indefinitely,
1701         * the maximum wait time is bounded by SCSI request timeout.
1702         */
1703        wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
1704
1705        init_completion(&wait);
1706        lrbp = &hba->lrb[tag];
1707        WARN_ON(lrbp->cmd);
1708        err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
1709        if (unlikely(err))
1710                goto out_put_tag;
1711
1712        hba->dev_cmd.complete = &wait;
1713
1714        /* Make sure descriptors are ready before ringing the doorbell */
1715        wmb();
1716        spin_lock_irqsave(hba->host->host_lock, flags);
1717        ufshcd_send_command(hba, tag);
1718        spin_unlock_irqrestore(hba->host->host_lock, flags);
1719
1720        err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
1721
1722out_put_tag:
1723        ufshcd_put_dev_cmd_tag(hba, tag);
1724        wake_up(&hba->dev_cmd.tag_wq);
1725        return err;
1726}
1727
1728/**
1729 * ufshcd_init_query() - init the query response and request parameters
1730 * @hba: per-adapter instance
1731 * @request: address of the request pointer to be initialized
1732 * @response: address of the response pointer to be initialized
1733 * @opcode: operation to perform
1734 * @idn: flag idn to access
1735 * @index: LU number to access
1736 * @selector: query/flag/descriptor further identification
1737 */
1738static inline void ufshcd_init_query(struct ufs_hba *hba,
1739                struct ufs_query_req **request, struct ufs_query_res **response,
1740                enum query_opcode opcode, u8 idn, u8 index, u8 selector)
1741{
1742        *request = &hba->dev_cmd.query.request;
1743        *response = &hba->dev_cmd.query.response;
1744        memset(*request, 0, sizeof(struct ufs_query_req));
1745        memset(*response, 0, sizeof(struct ufs_query_res));
1746        (*request)->upiu_req.opcode = opcode;
1747        (*request)->upiu_req.idn = idn;
1748        (*request)->upiu_req.index = index;
1749        (*request)->upiu_req.selector = selector;
1750}
1751
1752static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1753        enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
1754{
1755        int ret;
1756        int retries;
1757
1758        for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1759                ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
1760                if (ret)
1761                        dev_dbg(hba->dev,
1762                                "%s: failed with error %d, retries %d\n",
1763                                __func__, ret, retries);
1764                else
1765                        break;
1766        }
1767
1768        if (ret)
1769                dev_err(hba->dev,
1770                        "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1771                        __func__, opcode, idn, ret, retries);
1772        return ret;
1773}
1774
1775/**
1776 * ufshcd_query_flag() - API function for sending flag query requests
1777 * hba: per-adapter instance
1778 * query_opcode: flag query to perform
1779 * idn: flag idn to access
1780 * flag_res: the flag value after the query request completes
1781 *
1782 * Returns 0 for success, non-zero in case of failure
1783 */
1784int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1785                        enum flag_idn idn, bool *flag_res)
1786{
1787        struct ufs_query_req *request = NULL;
1788        struct ufs_query_res *response = NULL;
1789        int err, index = 0, selector = 0;
1790        int timeout = QUERY_REQ_TIMEOUT;
1791
1792        BUG_ON(!hba);
1793
1794        ufshcd_hold(hba, false);
1795        mutex_lock(&hba->dev_cmd.lock);
1796        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1797                        selector);
1798
1799        switch (opcode) {
1800        case UPIU_QUERY_OPCODE_SET_FLAG:
1801        case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1802        case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1803                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1804                break;
1805        case UPIU_QUERY_OPCODE_READ_FLAG:
1806                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1807                if (!flag_res) {
1808                        /* No dummy reads */
1809                        dev_err(hba->dev, "%s: Invalid argument for read request\n",
1810                                        __func__);
1811                        err = -EINVAL;
1812                        goto out_unlock;
1813                }
1814                break;
1815        default:
1816                dev_err(hba->dev,
1817                        "%s: Expected query flag opcode but got = %d\n",
1818                        __func__, opcode);
1819                err = -EINVAL;
1820                goto out_unlock;
1821        }
1822
1823        if (idn == QUERY_FLAG_IDN_FDEVICEINIT)
1824                timeout = QUERY_FDEVICEINIT_REQ_TIMEOUT;
1825
1826        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
1827
1828        if (err) {
1829                dev_err(hba->dev,
1830                        "%s: Sending flag query for idn %d failed, err = %d\n",
1831                        __func__, idn, err);
1832                goto out_unlock;
1833        }
1834
1835        if (flag_res)
1836                *flag_res = (be32_to_cpu(response->upiu_res.value) &
1837                                MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1838
1839out_unlock:
1840        mutex_unlock(&hba->dev_cmd.lock);
1841        ufshcd_release(hba);
1842        return err;
1843}
1844
1845/**
1846 * ufshcd_query_attr - API function for sending attribute requests
1847 * hba: per-adapter instance
1848 * opcode: attribute opcode
1849 * idn: attribute idn to access
1850 * index: index field
1851 * selector: selector field
1852 * attr_val: the attribute value after the query request completes
1853 *
1854 * Returns 0 for success, non-zero in case of failure
1855*/
1856static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1857                        enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
1858{
1859        struct ufs_query_req *request = NULL;
1860        struct ufs_query_res *response = NULL;
1861        int err;
1862
1863        BUG_ON(!hba);
1864
1865        ufshcd_hold(hba, false);
1866        if (!attr_val) {
1867                dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
1868                                __func__, opcode);
1869                err = -EINVAL;
1870                goto out;
1871        }
1872
1873        mutex_lock(&hba->dev_cmd.lock);
1874        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1875                        selector);
1876
1877        switch (opcode) {
1878        case UPIU_QUERY_OPCODE_WRITE_ATTR:
1879                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1880                request->upiu_req.value = cpu_to_be32(*attr_val);
1881                break;
1882        case UPIU_QUERY_OPCODE_READ_ATTR:
1883                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1884                break;
1885        default:
1886                dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
1887                                __func__, opcode);
1888                err = -EINVAL;
1889                goto out_unlock;
1890        }
1891
1892        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1893
1894        if (err) {
1895                dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1896                                __func__, opcode, idn, err);
1897                goto out_unlock;
1898        }
1899
1900        *attr_val = be32_to_cpu(response->upiu_res.value);
1901
1902out_unlock:
1903        mutex_unlock(&hba->dev_cmd.lock);
1904out:
1905        ufshcd_release(hba);
1906        return err;
1907}
1908
1909/**
1910 * ufshcd_query_attr_retry() - API function for sending query
1911 * attribute with retries
1912 * @hba: per-adapter instance
1913 * @opcode: attribute opcode
1914 * @idn: attribute idn to access
1915 * @index: index field
1916 * @selector: selector field
1917 * @attr_val: the attribute value after the query request
1918 * completes
1919 *
1920 * Returns 0 for success, non-zero in case of failure
1921*/
1922static int ufshcd_query_attr_retry(struct ufs_hba *hba,
1923        enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
1924        u32 *attr_val)
1925{
1926        int ret = 0;
1927        u32 retries;
1928
1929         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1930                ret = ufshcd_query_attr(hba, opcode, idn, index,
1931                                                selector, attr_val);
1932                if (ret)
1933                        dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
1934                                __func__, ret, retries);
1935                else
1936                        break;
1937        }
1938
1939        if (ret)
1940                dev_err(hba->dev,
1941                        "%s: query attribute, idn %d, failed with error %d after %d retires\n",
1942                        __func__, idn, ret, QUERY_REQ_RETRIES);
1943        return ret;
1944}
1945
1946static int __ufshcd_query_descriptor(struct ufs_hba *hba,
1947                        enum query_opcode opcode, enum desc_idn idn, u8 index,
1948                        u8 selector, u8 *desc_buf, int *buf_len)
1949{
1950        struct ufs_query_req *request = NULL;
1951        struct ufs_query_res *response = NULL;
1952        int err;
1953
1954        BUG_ON(!hba);
1955
1956        ufshcd_hold(hba, false);
1957        if (!desc_buf) {
1958                dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
1959                                __func__, opcode);
1960                err = -EINVAL;
1961                goto out;
1962        }
1963
1964        if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
1965                dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
1966                                __func__, *buf_len);
1967                err = -EINVAL;
1968                goto out;
1969        }
1970
1971        mutex_lock(&hba->dev_cmd.lock);
1972        ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1973                        selector);
1974        hba->dev_cmd.query.descriptor = desc_buf;
1975        request->upiu_req.length = cpu_to_be16(*buf_len);
1976
1977        switch (opcode) {
1978        case UPIU_QUERY_OPCODE_WRITE_DESC:
1979                request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1980                break;
1981        case UPIU_QUERY_OPCODE_READ_DESC:
1982                request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1983                break;
1984        default:
1985                dev_err(hba->dev,
1986                                "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1987                                __func__, opcode);
1988                err = -EINVAL;
1989                goto out_unlock;
1990        }
1991
1992        err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1993
1994        if (err) {
1995                dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1996                                __func__, opcode, idn, err);
1997                goto out_unlock;
1998        }
1999
2000        hba->dev_cmd.query.descriptor = NULL;
2001        *buf_len = be16_to_cpu(response->upiu_res.length);
2002
2003out_unlock:
2004        mutex_unlock(&hba->dev_cmd.lock);
2005out:
2006        ufshcd_release(hba);
2007        return err;
2008}
2009
2010/**
2011 * ufshcd_query_descriptor_retry - API function for sending descriptor
2012 * requests
2013 * hba: per-adapter instance
2014 * opcode: attribute opcode
2015 * idn: attribute idn to access
2016 * index: index field
2017 * selector: selector field
2018 * desc_buf: the buffer that contains the descriptor
2019 * buf_len: length parameter passed to the device
2020 *
2021 * Returns 0 for success, non-zero in case of failure.
2022 * The buf_len parameter will contain, on return, the length parameter
2023 * received on the response.
2024 */
2025int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
2026                        enum query_opcode opcode, enum desc_idn idn, u8 index,
2027                        u8 selector, u8 *desc_buf, int *buf_len)
2028{
2029        int err;
2030        int retries;
2031
2032        for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2033                err = __ufshcd_query_descriptor(hba, opcode, idn, index,
2034                                                selector, desc_buf, buf_len);
2035                if (!err || err == -EINVAL)
2036                        break;
2037        }
2038
2039        return err;
2040}
2041EXPORT_SYMBOL(ufshcd_query_descriptor_retry);
2042
2043/**
2044 * ufshcd_read_desc_param - read the specified descriptor parameter
2045 * @hba: Pointer to adapter instance
2046 * @desc_id: descriptor idn value
2047 * @desc_index: descriptor index
2048 * @param_offset: offset of the parameter to read
2049 * @param_read_buf: pointer to buffer where parameter would be read
2050 * @param_size: sizeof(param_read_buf)
2051 *
2052 * Return 0 in case of success, non-zero otherwise
2053 */
2054static int ufshcd_read_desc_param(struct ufs_hba *hba,
2055                                  enum desc_idn desc_id,
2056                                  int desc_index,
2057                                  u32 param_offset,
2058                                  u8 *param_read_buf,
2059                                  u32 param_size)
2060{
2061        int ret;
2062        u8 *desc_buf;
2063        u32 buff_len;
2064        bool is_kmalloc = true;
2065
2066        /* safety checks */
2067        if (desc_id >= QUERY_DESC_IDN_MAX)
2068                return -EINVAL;
2069
2070        buff_len = ufs_query_desc_max_size[desc_id];
2071        if ((param_offset + param_size) > buff_len)
2072                return -EINVAL;
2073
2074        if (!param_offset && (param_size == buff_len)) {
2075                /* memory space already available to hold full descriptor */
2076                desc_buf = param_read_buf;
2077                is_kmalloc = false;
2078        } else {
2079                /* allocate memory to hold full descriptor */
2080                desc_buf = kmalloc(buff_len, GFP_KERNEL);
2081                if (!desc_buf)
2082                        return -ENOMEM;
2083        }
2084
2085        ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2086                                        desc_id, desc_index, 0, desc_buf,
2087                                        &buff_len);
2088
2089        if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) ||
2090            (desc_buf[QUERY_DESC_LENGTH_OFFSET] !=
2091             ufs_query_desc_max_size[desc_id])
2092            || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) {
2093                dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d",
2094                        __func__, desc_id, param_offset, buff_len, ret);
2095                if (!ret)
2096                        ret = -EINVAL;
2097
2098                goto out;
2099        }
2100
2101        if (is_kmalloc)
2102                memcpy(param_read_buf, &desc_buf[param_offset], param_size);
2103out:
2104        if (is_kmalloc)
2105                kfree(desc_buf);
2106        return ret;
2107}
2108
2109static inline int ufshcd_read_desc(struct ufs_hba *hba,
2110                                   enum desc_idn desc_id,
2111                                   int desc_index,
2112                                   u8 *buf,
2113                                   u32 size)
2114{
2115        return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
2116}
2117
2118static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
2119                                         u8 *buf,
2120                                         u32 size)
2121{
2122        return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
2123}
2124
2125int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
2126{
2127        return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
2128}
2129EXPORT_SYMBOL(ufshcd_read_device_desc);
2130
2131/**
2132 * ufshcd_read_string_desc - read string descriptor
2133 * @hba: pointer to adapter instance
2134 * @desc_index: descriptor index
2135 * @buf: pointer to buffer where descriptor would be read
2136 * @size: size of buf
2137 * @ascii: if true convert from unicode to ascii characters
2138 *
2139 * Return 0 in case of success, non-zero otherwise
2140 */
2141int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, u8 *buf,
2142                                u32 size, bool ascii)
2143{
2144        int err = 0;
2145
2146        err = ufshcd_read_desc(hba,
2147                                QUERY_DESC_IDN_STRING, desc_index, buf, size);
2148
2149        if (err) {
2150                dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
2151                        __func__, QUERY_REQ_RETRIES, err);
2152                goto out;
2153        }
2154
2155        if (ascii) {
2156                int desc_len;
2157                int ascii_len;
2158                int i;
2159                char *buff_ascii;
2160
2161                desc_len = buf[0];
2162                /* remove header and divide by 2 to move from UTF16 to UTF8 */
2163                ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
2164                if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
2165                        dev_err(hba->dev, "%s: buffer allocated size is too small\n",
2166                                        __func__);
2167                        err = -ENOMEM;
2168                        goto out;
2169                }
2170
2171                buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
2172                if (!buff_ascii) {
2173                        err = -ENOMEM;
2174                        goto out;
2175                }
2176
2177                /*
2178                 * the descriptor contains string in UTF16 format
2179                 * we need to convert to utf-8 so it can be displayed
2180                 */
2181                utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
2182                                desc_len - QUERY_DESC_HDR_SIZE,
2183                                UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
2184
2185                /* replace non-printable or non-ASCII characters with spaces */
2186                for (i = 0; i < ascii_len; i++)
2187                        ufshcd_remove_non_printable(&buff_ascii[i]);
2188
2189                memset(buf + QUERY_DESC_HDR_SIZE, 0,
2190                                size - QUERY_DESC_HDR_SIZE);
2191                memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
2192                buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
2193                kfree(buff_ascii);
2194        }
2195out:
2196        return err;
2197}
2198EXPORT_SYMBOL(ufshcd_read_string_desc);
2199
2200/**
2201 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
2202 * @hba: Pointer to adapter instance
2203 * @lun: lun id
2204 * @param_offset: offset of the parameter to read
2205 * @param_read_buf: pointer to buffer where parameter would be read
2206 * @param_size: sizeof(param_read_buf)
2207 *
2208 * Return 0 in case of success, non-zero otherwise
2209 */
2210static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
2211                                              int lun,
2212                                              enum unit_desc_param param_offset,
2213                                              u8 *param_read_buf,
2214                                              u32 param_size)
2215{
2216        /*
2217         * Unit descriptors are only available for general purpose LUs (LUN id
2218         * from 0 to 7) and RPMB Well known LU.
2219         */
2220        if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN))
2221                return -EOPNOTSUPP;
2222
2223        return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
2224                                      param_offset, param_read_buf, param_size);
2225}
2226
2227/**
2228 * ufshcd_memory_alloc - allocate memory for host memory space data structures
2229 * @hba: per adapter instance
2230 *
2231 * 1. Allocate DMA memory for Command Descriptor array
2232 *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
2233 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
2234 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
2235 *      (UTMRDL)
2236 * 4. Allocate memory for local reference block(lrb).
2237 *
2238 * Returns 0 for success, non-zero in case of failure
2239 */
2240static int ufshcd_memory_alloc(struct ufs_hba *hba)
2241{
2242        size_t utmrdl_size, utrdl_size, ucdl_size;
2243
2244        /* Allocate memory for UTP command descriptors */
2245        ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2246        hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
2247                                                  ucdl_size,
2248                                                  &hba->ucdl_dma_addr,
2249                                                  GFP_KERNEL);
2250
2251        /*
2252         * UFSHCI requires UTP command descriptor to be 128 byte aligned.
2253         * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
2254         * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
2255         * be aligned to 128 bytes as well
2256         */
2257        if (!hba->ucdl_base_addr ||
2258            WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
2259                dev_err(hba->dev,
2260                        "Command Descriptor Memory allocation failed\n");
2261                goto out;
2262        }
2263
2264        /*
2265         * Allocate memory for UTP Transfer descriptors
2266         * UFSHCI requires 1024 byte alignment of UTRD
2267         */
2268        utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2269        hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
2270                                                   utrdl_size,
2271                                                   &hba->utrdl_dma_addr,
2272                                                   GFP_KERNEL);
2273        if (!hba->utrdl_base_addr ||
2274            WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
2275                dev_err(hba->dev,
2276                        "Transfer Descriptor Memory allocation failed\n");
2277                goto out;
2278        }
2279
2280        /*
2281         * Allocate memory for UTP Task Management descriptors
2282         * UFSHCI requires 1024 byte alignment of UTMRD
2283         */
2284        utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2285        hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
2286                                                    utmrdl_size,
2287                                                    &hba->utmrdl_dma_addr,
2288                                                    GFP_KERNEL);
2289        if (!hba->utmrdl_base_addr ||
2290            WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
2291                dev_err(hba->dev,
2292                "Task Management Descriptor Memory allocation failed\n");
2293                goto out;
2294        }
2295
2296        /* Allocate memory for local reference block */
2297        hba->lrb = devm_kzalloc(hba->dev,
2298                                hba->nutrs * sizeof(struct ufshcd_lrb),
2299                                GFP_KERNEL);
2300        if (!hba->lrb) {
2301                dev_err(hba->dev, "LRB Memory allocation failed\n");
2302                goto out;
2303        }
2304        return 0;
2305out:
2306        return -ENOMEM;
2307}
2308
2309/**
2310 * ufshcd_host_memory_configure - configure local reference block with
2311 *                              memory offsets
2312 * @hba: per adapter instance
2313 *
2314 * Configure Host memory space
2315 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
2316 * address.
2317 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
2318 * and PRDT offset.
2319 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
2320 * into local reference block.
2321 */
2322static void ufshcd_host_memory_configure(struct ufs_hba *hba)
2323{
2324        struct utp_transfer_cmd_desc *cmd_descp;
2325        struct utp_transfer_req_desc *utrdlp;
2326        dma_addr_t cmd_desc_dma_addr;
2327        dma_addr_t cmd_desc_element_addr;
2328        u16 response_offset;
2329        u16 prdt_offset;
2330        int cmd_desc_size;
2331        int i;
2332
2333        utrdlp = hba->utrdl_base_addr;
2334        cmd_descp = hba->ucdl_base_addr;
2335
2336        response_offset =
2337                offsetof(struct utp_transfer_cmd_desc, response_upiu);
2338        prdt_offset =
2339                offsetof(struct utp_transfer_cmd_desc, prd_table);
2340
2341        cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
2342        cmd_desc_dma_addr = hba->ucdl_dma_addr;
2343
2344        for (i = 0; i < hba->nutrs; i++) {
2345                /* Configure UTRD with command descriptor base address */
2346                cmd_desc_element_addr =
2347                                (cmd_desc_dma_addr + (cmd_desc_size * i));
2348                utrdlp[i].command_desc_base_addr_lo =
2349                                cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
2350                utrdlp[i].command_desc_base_addr_hi =
2351                                cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
2352
2353                /* Response upiu and prdt offset should be in double words */
2354                if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
2355                        utrdlp[i].response_upiu_offset =
2356                                cpu_to_le16(response_offset);
2357                        utrdlp[i].prd_table_offset =
2358                                cpu_to_le16(prdt_offset);
2359                        utrdlp[i].response_upiu_length =
2360                                cpu_to_le16(ALIGNED_UPIU_SIZE);
2361                } else {
2362                        utrdlp[i].response_upiu_offset =
2363                                cpu_to_le16((response_offset >> 2));
2364                        utrdlp[i].prd_table_offset =
2365                                cpu_to_le16((prdt_offset >> 2));
2366                        utrdlp[i].response_upiu_length =
2367                                cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
2368                }
2369
2370                hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
2371                hba->lrb[i].ucd_req_ptr =
2372                        (struct utp_upiu_req *)(cmd_descp + i);
2373                hba->lrb[i].ucd_rsp_ptr =
2374                        (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2375                hba->lrb[i].ucd_prdt_ptr =
2376                        (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2377        }
2378}
2379
2380/**
2381 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
2382 * @hba: per adapter instance
2383 *
2384 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
2385 * in order to initialize the Unipro link startup procedure.
2386 * Once the Unipro links are up, the device connected to the controller
2387 * is detected.
2388 *
2389 * Returns 0 on success, non-zero value on failure
2390 */
2391static int ufshcd_dme_link_startup(struct ufs_hba *hba)
2392{
2393        struct uic_command uic_cmd = {0};
2394        int ret;
2395
2396        uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
2397
2398        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2399        if (ret)
2400                dev_err(hba->dev,
2401                        "dme-link-startup: error code %d\n", ret);
2402        return ret;
2403}
2404
2405static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
2406{
2407        #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
2408        unsigned long min_sleep_time_us;
2409
2410        if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
2411                return;
2412
2413        /*
2414         * last_dme_cmd_tstamp will be 0 only for 1st call to
2415         * this function
2416         */
2417        if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
2418                min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
2419        } else {
2420                unsigned long delta =
2421                        (unsigned long) ktime_to_us(
2422                                ktime_sub(ktime_get(),
2423                                hba->last_dme_cmd_tstamp));
2424
2425                if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
2426                        min_sleep_time_us =
2427                                MIN_DELAY_BEFORE_DME_CMDS_US - delta;
2428                else
2429                        return; /* no more delay required */
2430        }
2431
2432        /* allow sleep for extra 50us if needed */
2433        usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
2434}
2435
2436/**
2437 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
2438 * @hba: per adapter instance
2439 * @attr_sel: uic command argument1
2440 * @attr_set: attribute set type as uic command argument2
2441 * @mib_val: setting value as uic command argument3
2442 * @peer: indicate whether peer or local
2443 *
2444 * Returns 0 on success, non-zero value on failure
2445 */
2446int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
2447                        u8 attr_set, u32 mib_val, u8 peer)
2448{
2449        struct uic_command uic_cmd = {0};
2450        static const char *const action[] = {
2451                "dme-set",
2452                "dme-peer-set"
2453        };
2454        const char *set = action[!!peer];
2455        int ret;
2456        int retries = UFS_UIC_COMMAND_RETRIES;
2457
2458        uic_cmd.command = peer ?
2459                UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
2460        uic_cmd.argument1 = attr_sel;
2461        uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
2462        uic_cmd.argument3 = mib_val;
2463
2464        do {
2465                /* for peer attributes we retry upon failure */
2466                ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2467                if (ret)
2468                        dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
2469                                set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
2470        } while (ret && peer && --retries);
2471
2472        if (!retries)
2473                dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
2474                                set, UIC_GET_ATTR_ID(attr_sel), mib_val,
2475                                retries);
2476
2477        return ret;
2478}
2479EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
2480
2481/**
2482 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
2483 * @hba: per adapter instance
2484 * @attr_sel: uic command argument1
2485 * @mib_val: the value of the attribute as returned by the UIC command
2486 * @peer: indicate whether peer or local
2487 *
2488 * Returns 0 on success, non-zero value on failure
2489 */
2490int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
2491                        u32 *mib_val, u8 peer)
2492{
2493        struct uic_command uic_cmd = {0};
2494        static const char *const action[] = {
2495                "dme-get",
2496                "dme-peer-get"
2497        };
2498        const char *get = action[!!peer];
2499        int ret;
2500        int retries = UFS_UIC_COMMAND_RETRIES;
2501        struct ufs_pa_layer_attr orig_pwr_info;
2502        struct ufs_pa_layer_attr temp_pwr_info;
2503        bool pwr_mode_change = false;
2504
2505        if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
2506                orig_pwr_info = hba->pwr_info;
2507                temp_pwr_info = orig_pwr_info;
2508
2509                if (orig_pwr_info.pwr_tx == FAST_MODE ||
2510                    orig_pwr_info.pwr_rx == FAST_MODE) {
2511                        temp_pwr_info.pwr_tx = FASTAUTO_MODE;
2512                        temp_pwr_info.pwr_rx = FASTAUTO_MODE;
2513                        pwr_mode_change = true;
2514                } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
2515                    orig_pwr_info.pwr_rx == SLOW_MODE) {
2516                        temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
2517                        temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
2518                        pwr_mode_change = true;
2519                }
2520                if (pwr_mode_change) {
2521                        ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
2522                        if (ret)
2523                                goto out;
2524                }
2525        }
2526
2527        uic_cmd.command = peer ?
2528                UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
2529        uic_cmd.argument1 = attr_sel;
2530
2531        do {
2532                /* for peer attributes we retry upon failure */
2533                ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2534                if (ret)
2535                        dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
2536                                get, UIC_GET_ATTR_ID(attr_sel), ret);
2537        } while (ret && peer && --retries);
2538
2539        if (!retries)
2540                dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
2541                                get, UIC_GET_ATTR_ID(attr_sel), retries);
2542
2543        if (mib_val && !ret)
2544                *mib_val = uic_cmd.argument3;
2545
2546        if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
2547            && pwr_mode_change)
2548                ufshcd_change_power_mode(hba, &orig_pwr_info);
2549out:
2550        return ret;
2551}
2552EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
2553
2554/**
2555 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
2556 * state) and waits for it to take effect.
2557 *
2558 * @hba: per adapter instance
2559 * @cmd: UIC command to execute
2560 *
2561 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
2562 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
2563 * and device UniPro link and hence it's final completion would be indicated by
2564 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
2565 * addition to normal UIC command completion Status (UCCS). This function only
2566 * returns after the relevant status bits indicate the completion.
2567 *
2568 * Returns 0 on success, non-zero value on failure
2569 */
2570static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
2571{
2572        struct completion uic_async_done;
2573        unsigned long flags;
2574        u8 status;
2575        int ret;
2576        bool reenable_intr = false;
2577
2578        mutex_lock(&hba->uic_cmd_mutex);
2579        init_completion(&uic_async_done);
2580        ufshcd_add_delay_before_dme_cmd(hba);
2581
2582        spin_lock_irqsave(hba->host->host_lock, flags);
2583        hba->uic_async_done = &uic_async_done;
2584        if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
2585                ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
2586                /*
2587                 * Make sure UIC command completion interrupt is disabled before
2588                 * issuing UIC command.
2589                 */
2590                wmb();
2591                reenable_intr = true;
2592        }
2593        ret = __ufshcd_send_uic_cmd(hba, cmd, false);
2594        spin_unlock_irqrestore(hba->host->host_lock, flags);
2595        if (ret) {
2596                dev_err(hba->dev,
2597                        "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
2598                        cmd->command, cmd->argument3, ret);
2599                goto out;
2600        }
2601
2602        if (!wait_for_completion_timeout(hba->uic_async_done,
2603                                         msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2604                dev_err(hba->dev,
2605                        "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
2606                        cmd->command, cmd->argument3);
2607                ret = -ETIMEDOUT;
2608                goto out;
2609        }
2610
2611        status = ufshcd_get_upmcrs(hba);
2612        if (status != PWR_LOCAL) {
2613                dev_err(hba->dev,
2614                        "pwr ctrl cmd 0x%0x failed, host upmcrs:0x%x\n",
2615                        cmd->command, status);
2616                ret = (status != PWR_OK) ? status : -1;
2617        }
2618out:
2619        spin_lock_irqsave(hba->host->host_lock, flags);
2620        hba->active_uic_cmd = NULL;
2621        hba->uic_async_done = NULL;
2622        if (reenable_intr)
2623                ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
2624        spin_unlock_irqrestore(hba->host->host_lock, flags);
2625        mutex_unlock(&hba->uic_cmd_mutex);
2626
2627        return ret;
2628}
2629
2630/**
2631 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
2632 *                              using DME_SET primitives.
2633 * @hba: per adapter instance
2634 * @mode: powr mode value
2635 *
2636 * Returns 0 on success, non-zero value on failure
2637 */
2638static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
2639{
2640        struct uic_command uic_cmd = {0};
2641        int ret;
2642
2643        if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
2644                ret = ufshcd_dme_set(hba,
2645                                UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
2646                if (ret) {
2647                        dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
2648                                                __func__, ret);
2649                        goto out;
2650                }
2651        }
2652
2653        uic_cmd.command = UIC_CMD_DME_SET;
2654        uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
2655        uic_cmd.argument3 = mode;
2656        ufshcd_hold(hba, false);
2657        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2658        ufshcd_release(hba);
2659
2660out:
2661        return ret;
2662}
2663
2664static int ufshcd_link_recovery(struct ufs_hba *hba)
2665{
2666        int ret;
2667        unsigned long flags;
2668
2669        spin_lock_irqsave(hba->host->host_lock, flags);
2670        hba->ufshcd_state = UFSHCD_STATE_RESET;
2671        ufshcd_set_eh_in_progress(hba);
2672        spin_unlock_irqrestore(hba->host->host_lock, flags);
2673
2674        ret = ufshcd_host_reset_and_restore(hba);
2675
2676        spin_lock_irqsave(hba->host->host_lock, flags);
2677        if (ret)
2678                hba->ufshcd_state = UFSHCD_STATE_ERROR;
2679        ufshcd_clear_eh_in_progress(hba);
2680        spin_unlock_irqrestore(hba->host->host_lock, flags);
2681
2682        if (ret)
2683                dev_err(hba->dev, "%s: link recovery failed, err %d",
2684                        __func__, ret);
2685
2686        return ret;
2687}
2688
2689static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2690{
2691        int ret;
2692        struct uic_command uic_cmd = {0};
2693
2694        uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
2695        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2696
2697        if (ret) {
2698                dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
2699                        __func__, ret);
2700
2701                /*
2702                 * If link recovery fails then return error so that caller
2703                 * don't retry the hibern8 enter again.
2704                 */
2705                if (ufshcd_link_recovery(hba))
2706                        ret = -ENOLINK;
2707        }
2708
2709        return ret;
2710}
2711
2712static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2713{
2714        int ret = 0, retries;
2715
2716        for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
2717                ret = __ufshcd_uic_hibern8_enter(hba);
2718                if (!ret || ret == -ENOLINK)
2719                        goto out;
2720        }
2721out:
2722        return ret;
2723}
2724
2725static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
2726{
2727        struct uic_command uic_cmd = {0};
2728        int ret;
2729
2730        uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
2731        ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2732        if (ret) {
2733                dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
2734                        __func__, ret);
2735                ret = ufshcd_link_recovery(hba);
2736        }
2737
2738        return ret;
2739}
2740
2741 /**
2742 * ufshcd_init_pwr_info - setting the POR (power on reset)
2743 * values in hba power info
2744 * @hba: per-adapter instance
2745 */
2746static void ufshcd_init_pwr_info(struct ufs_hba *hba)
2747{
2748        hba->pwr_info.gear_rx = UFS_PWM_G1;
2749        hba->pwr_info.gear_tx = UFS_PWM_G1;
2750        hba->pwr_info.lane_rx = 1;
2751        hba->pwr_info.lane_tx = 1;
2752        hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
2753        hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
2754        hba->pwr_info.hs_rate = 0;
2755}
2756
2757/**
2758 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
2759 * @hba: per-adapter instance
2760 */
2761static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
2762{
2763        struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
2764
2765        if (hba->max_pwr_info.is_valid)
2766                return 0;
2767
2768        pwr_info->pwr_tx = FASTAUTO_MODE;
2769        pwr_info->pwr_rx = FASTAUTO_MODE;
2770        pwr_info->hs_rate = PA_HS_MODE_B;
2771
2772        /* Get the connected lane count */
2773        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
2774                        &pwr_info->lane_rx);
2775        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
2776                        &pwr_info->lane_tx);
2777
2778        if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
2779                dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
2780                                __func__,
2781                                pwr_info->lane_rx,
2782                                pwr_info->lane_tx);
2783                return -EINVAL;
2784        }
2785
2786        /*
2787         * First, get the maximum gears of HS speed.
2788         * If a zero value, it means there is no HSGEAR capability.
2789         * Then, get the maximum gears of PWM speed.
2790         */
2791        ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
2792        if (!pwr_info->gear_rx) {
2793                ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2794                                &pwr_info->gear_rx);
2795                if (!pwr_info->gear_rx) {
2796                        dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
2797                                __func__, pwr_info->gear_rx);
2798                        return -EINVAL;
2799                }
2800                pwr_info->pwr_rx = SLOWAUTO_MODE;
2801        }
2802
2803        ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
2804                        &pwr_info->gear_tx);
2805        if (!pwr_info->gear_tx) {
2806                ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2807                                &pwr_info->gear_tx);
2808                if (!pwr_info->gear_tx) {
2809                        dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
2810                                __func__, pwr_info->gear_tx);
2811                        return -EINVAL;
2812                }
2813                pwr_info->pwr_tx = SLOWAUTO_MODE;
2814        }
2815
2816        hba->max_pwr_info.is_valid = true;
2817        return 0;
2818}
2819
2820static int ufshcd_change_power_mode(struct ufs_hba *hba,
2821                             struct ufs_pa_layer_attr *pwr_mode)
2822{
2823        int ret;
2824
2825        /* if already configured to the requested pwr_mode */
2826        if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
2827            pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
2828            pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
2829            pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
2830            pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
2831            pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
2832            pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
2833                dev_dbg(hba->dev, "%s: power already configured\n", __func__);
2834                return 0;
2835        }
2836
2837        /*
2838         * Configure attributes for power mode change with below.
2839         * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
2840         * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
2841         * - PA_HSSERIES
2842         */
2843        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
2844        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
2845                        pwr_mode->lane_rx);
2846        if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2847                        pwr_mode->pwr_rx == FAST_MODE)
2848                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
2849        else
2850                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
2851
2852        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
2853        ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
2854                        pwr_mode->lane_tx);
2855        if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
2856                        pwr_mode->pwr_tx == FAST_MODE)
2857                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
2858        else
2859                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
2860
2861        if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2862            pwr_mode->pwr_tx == FASTAUTO_MODE ||
2863            pwr_mode->pwr_rx == FAST_MODE ||
2864            pwr_mode->pwr_tx == FAST_MODE)
2865                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
2866                                                pwr_mode->hs_rate);
2867
2868        ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
2869                        | pwr_mode->pwr_tx);
2870
2871        if (ret) {
2872                dev_err(hba->dev,
2873                        "%s: power mode change failed %d\n", __func__, ret);
2874        } else {
2875                ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
2876                                                                pwr_mode);
2877
2878                memcpy(&hba->pwr_info, pwr_mode,
2879                        sizeof(struct ufs_pa_layer_attr));
2880        }
2881
2882        return ret;
2883}
2884
2885/**
2886 * ufshcd_config_pwr_mode - configure a new power mode
2887 * @hba: per-adapter instance
2888 * @desired_pwr_mode: desired power configuration
2889 */
2890static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
2891                struct ufs_pa_layer_attr *desired_pwr_mode)
2892{
2893        struct ufs_pa_layer_attr final_params = { 0 };
2894        int ret;
2895
2896        ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
2897                                        desired_pwr_mode, &final_params);
2898
2899        if (ret)
2900                memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
2901
2902        ret = ufshcd_change_power_mode(hba, &final_params);
2903
2904        return ret;
2905}
2906
2907/**
2908 * ufshcd_complete_dev_init() - checks device readiness
2909 * hba: per-adapter instance
2910 *
2911 * Set fDeviceInit flag and poll until device toggles it.
2912 */
2913static int ufshcd_complete_dev_init(struct ufs_hba *hba)
2914{
2915        int i;
2916        int err;
2917        bool flag_res = 1;
2918
2919        err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
2920                QUERY_FLAG_IDN_FDEVICEINIT, NULL);
2921        if (err) {
2922                dev_err(hba->dev,
2923                        "%s setting fDeviceInit flag failed with error %d\n",
2924                        __func__, err);
2925                goto out;
2926        }
2927
2928        /* poll for max. 1000 iterations for fDeviceInit flag to clear */
2929        for (i = 0; i < 1000 && !err && flag_res; i++)
2930                err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
2931                        QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
2932
2933        if (err)
2934                dev_err(hba->dev,
2935                        "%s reading fDeviceInit flag failed with error %d\n",
2936                        __func__, err);
2937        else if (flag_res)
2938                dev_err(hba->dev,
2939                        "%s fDeviceInit was not cleared by the device\n",
2940                        __func__);
2941
2942out:
2943        return err;
2944}
2945
2946/**
2947 * ufshcd_make_hba_operational - Make UFS controller operational
2948 * @hba: per adapter instance
2949 *
2950 * To bring UFS host controller to operational state,
2951 * 1. Enable required interrupts
2952 * 2. Configure interrupt aggregation
2953 * 3. Program UTRL and UTMRL base address
2954 * 4. Configure run-stop-registers
2955 *
2956 * Returns 0 on success, non-zero value on failure
2957 */
2958static int ufshcd_make_hba_operational(struct ufs_hba *hba)
2959{
2960        int err = 0;
2961        u32 reg;
2962
2963        /* Enable required interrupts */
2964        ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
2965
2966        /* Configure interrupt aggregation */
2967        if (ufshcd_is_intr_aggr_allowed(hba))
2968                ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
2969        else
2970                ufshcd_disable_intr_aggr(hba);
2971
2972        /* Configure UTRL and UTMRL base address registers */
2973        ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
2974                        REG_UTP_TRANSFER_REQ_LIST_BASE_L);
2975        ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
2976                        REG_UTP_TRANSFER_REQ_LIST_BASE_H);
2977        ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
2978                        REG_UTP_TASK_REQ_LIST_BASE_L);
2979        ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
2980                        REG_UTP_TASK_REQ_LIST_BASE_H);
2981
2982        /*
2983         * Make sure base address and interrupt setup are updated before
2984         * enabling the run/stop registers below.
2985         */
2986        wmb();
2987
2988        /*
2989         * UCRDY, UTMRLDY and UTRLRDY bits must be 1
2990         */
2991        reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
2992        if (!(ufshcd_get_lists_status(reg))) {
2993                ufshcd_enable_run_stop_reg(hba);
2994        } else {
2995                dev_err(hba->dev,
2996                        "Host controller not ready to process requests");
2997                err = -EIO;
2998                goto out;
2999        }
3000
3001out:
3002        return err;
3003}
3004
3005/**
3006 * ufshcd_hba_stop - Send controller to reset state
3007 * @hba: per adapter instance
3008 * @can_sleep: perform sleep or just spin
3009 */
3010static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
3011{
3012        int err;
3013
3014        ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
3015        err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
3016                                        CONTROLLER_ENABLE, CONTROLLER_DISABLE,
3017                                        10, 1, can_sleep);
3018        if (err)
3019                dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
3020}
3021
3022/**
3023 * ufshcd_hba_enable - initialize the controller
3024 * @hba: per adapter instance
3025 *
3026 * The controller resets itself and controller firmware initialization
3027 * sequence kicks off. When controller is ready it will set
3028 * the Host Controller Enable bit to 1.
3029 *
3030 * Returns 0 on success, non-zero value on failure
3031 */
3032static int ufshcd_hba_enable(struct ufs_hba *hba)
3033{
3034        int retry;
3035
3036        /*
3037         * msleep of 1 and 5 used in this function might result in msleep(20),
3038         * but it was necessary to send the UFS FPGA to reset mode during
3039         * development and testing of this driver. msleep can be changed to
3040         * mdelay and retry count can be reduced based on the controller.
3041         */
3042        if (!ufshcd_is_hba_active(hba))
3043                /* change controller state to "reset state" */
3044                ufshcd_hba_stop(hba, true);
3045
3046        /* UniPro link is disabled at this point */
3047        ufshcd_set_link_off(hba);
3048
3049        ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
3050
3051        /* start controller initialization sequence */
3052        ufshcd_hba_start(hba);
3053
3054        /*
3055         * To initialize a UFS host controller HCE bit must be set to 1.
3056         * During initialization the HCE bit value changes from 1->0->1.
3057         * When the host controller completes initialization sequence
3058         * it sets the value of HCE bit to 1. The same HCE bit is read back
3059         * to check if the controller has completed initialization sequence.
3060         * So without this delay the value HCE = 1, set in the previous
3061         * instruction might be read back.
3062         * This delay can be changed based on the controller.
3063         */
3064        msleep(1);
3065
3066        /* wait for the host controller to complete initialization */
3067        retry = 10;
3068        while (ufshcd_is_hba_active(hba)) {
3069                if (retry) {
3070                        retry--;
3071                } else {
3072                        dev_err(hba->dev,
3073                                "Controller enable failed\n");
3074                        return -EIO;
3075                }
3076                msleep(5);
3077        }
3078
3079        /* enable UIC related interrupts */
3080        ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
3081
3082        ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
3083
3084        return 0;
3085}
3086
3087static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
3088{
3089        int tx_lanes, i, err = 0;
3090
3091        if (!peer)
3092                ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3093                               &tx_lanes);
3094        else
3095                ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3096                                    &tx_lanes);
3097        for (i = 0; i < tx_lanes; i++) {
3098                if (!peer)
3099                        err = ufshcd_dme_set(hba,
3100                                UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3101                                        UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3102                                        0);
3103                else
3104                        err = ufshcd_dme_peer_set(hba,
3105                                UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3106                                        UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3107                                        0);
3108                if (err) {
3109                        dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
3110                                __func__, peer, i, err);
3111                        break;
3112                }
3113        }
3114
3115        return err;
3116}
3117
3118static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
3119{
3120        return ufshcd_disable_tx_lcc(hba, true);
3121}
3122
3123/**
3124 * ufshcd_link_startup - Initialize unipro link startup
3125 * @hba: per adapter instance
3126 *
3127 * Returns 0 for success, non-zero in case of failure
3128 */
3129static int ufshcd_link_startup(struct ufs_hba *hba)
3130{
3131        int ret;
3132        int retries = DME_LINKSTARTUP_RETRIES;
3133        bool link_startup_again = false;
3134
3135        /*
3136         * If UFS device isn't active then we will have to issue link startup
3137         * 2 times to make sure the device state move to active.
3138         */
3139        if (!ufshcd_is_ufs_dev_active(hba))
3140                link_startup_again = true;
3141
3142link_startup:
3143        do {
3144                ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
3145
3146                ret = ufshcd_dme_link_startup(hba);
3147
3148                /* check if device is detected by inter-connect layer */
3149                if (!ret && !ufshcd_is_device_present(hba)) {
3150                        dev_err(hba->dev, "%s: Device not present\n", __func__);
3151                        ret = -ENXIO;
3152                        goto out;
3153                }
3154
3155                /*
3156                 * DME link lost indication is only received when link is up,
3157                 * but we can't be sure if the link is up until link startup
3158                 * succeeds. So reset the local Uni-Pro and try again.
3159                 */
3160                if (ret && ufshcd_hba_enable(hba))
3161                        goto out;
3162        } while (ret && retries--);
3163
3164        if (ret)
3165                /* failed to get the link up... retire */
3166                goto out;
3167
3168        if (link_startup_again) {
3169                link_startup_again = false;
3170                retries = DME_LINKSTARTUP_RETRIES;
3171                goto link_startup;
3172        }
3173
3174        if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
3175                ret = ufshcd_disable_device_tx_lcc(hba);
3176                if (ret)
3177                        goto out;
3178        }
3179
3180        /* Include any host controller configuration via UIC commands */
3181        ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
3182        if (ret)
3183                goto out;
3184
3185        ret = ufshcd_make_hba_operational(hba);
3186out:
3187        if (ret)
3188                dev_err(hba->dev, "link startup failed %d\n", ret);
3189        return ret;
3190}
3191
3192/**
3193 * ufshcd_verify_dev_init() - Verify device initialization
3194 * @hba: per-adapter instance
3195 *
3196 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
3197 * device Transport Protocol (UTP) layer is ready after a reset.
3198 * If the UTP layer at the device side is not initialized, it may
3199 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
3200 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
3201 */
3202static int ufshcd_verify_dev_init(struct ufs_hba *hba)
3203{
3204        int err = 0;
3205        int retries;
3206
3207        ufshcd_hold(hba, false);
3208        mutex_lock(&hba->dev_cmd.lock);
3209        for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
3210                err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
3211                                               NOP_OUT_TIMEOUT);
3212
3213                if (!err || err == -ETIMEDOUT)
3214                        break;
3215
3216                dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
3217        }
3218        mutex_unlock(&hba->dev_cmd.lock);
3219        ufshcd_release(hba);
3220
3221        if (err)
3222                dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
3223        return err;
3224}
3225
3226/**
3227 * ufshcd_set_queue_depth - set lun queue depth
3228 * @sdev: pointer to SCSI device
3229 *
3230 * Read bLUQueueDepth value and activate scsi tagged command
3231 * queueing. For WLUN, queue depth is set to 1. For best-effort
3232 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
3233 * value that host can queue.
3234 */
3235static void ufshcd_set_queue_depth(struct scsi_device *sdev)
3236{
3237        int ret = 0;
3238        u8 lun_qdepth;
3239        struct ufs_hba *hba;
3240
3241        hba = shost_priv(sdev->host);
3242
3243        lun_qdepth = hba->nutrs;
3244        ret = ufshcd_read_unit_desc_param(hba,
3245                                          ufshcd_scsi_to_upiu_lun(sdev->lun),
3246                                          UNIT_DESC_PARAM_LU_Q_DEPTH,
3247                                          &lun_qdepth,
3248                                          sizeof(lun_qdepth));
3249
3250        /* Some WLUN doesn't support unit descriptor */
3251        if (ret == -EOPNOTSUPP)
3252                lun_qdepth = 1;
3253        else if (!lun_qdepth)
3254                /* eventually, we can figure out the real queue depth */
3255                lun_qdepth = hba->nutrs;
3256        else
3257                lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
3258
3259        dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
3260                        __func__, lun_qdepth);
3261        scsi_change_queue_depth(sdev, lun_qdepth);
3262}
3263
3264/*
3265 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
3266 * @hba: per-adapter instance
3267 * @lun: UFS device lun id
3268 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
3269 *
3270 * Returns 0 in case of success and b_lu_write_protect status would be returned
3271 * @b_lu_write_protect parameter.
3272 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
3273 * Returns -EINVAL in case of invalid parameters passed to this function.
3274 */
3275static int ufshcd_get_lu_wp(struct ufs_hba *hba,
3276                            u8 lun,
3277                            u8 *b_lu_write_protect)
3278{
3279        int ret;
3280
3281        if (!b_lu_write_protect)
3282                ret = -EINVAL;
3283        /*
3284         * According to UFS device spec, RPMB LU can't be write
3285         * protected so skip reading bLUWriteProtect parameter for
3286         * it. For other W-LUs, UNIT DESCRIPTOR is not available.
3287         */
3288        else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
3289                ret = -ENOTSUPP;
3290        else
3291                ret = ufshcd_read_unit_desc_param(hba,
3292                                          lun,
3293                                          UNIT_DESC_PARAM_LU_WR_PROTECT,
3294                                          b_lu_write_protect,
3295                                          sizeof(*b_lu_write_protect));
3296        return ret;
3297}
3298
3299/**
3300 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
3301 * status
3302 * @hba: per-adapter instance
3303 * @sdev: pointer to SCSI device
3304 *
3305 */
3306static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
3307                                                    struct scsi_device *sdev)
3308{
3309        if (hba->dev_info.f_power_on_wp_en &&
3310            !hba->dev_info.is_lu_power_on_wp) {
3311                u8 b_lu_write_protect;
3312
3313                if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
3314                                      &b_lu_write_protect) &&
3315                    (b_lu_write_protect == UFS_LU_POWER_ON_WP))
3316                        hba->dev_info.is_lu_power_on_wp = true;
3317        }
3318}
3319
3320/**
3321 * ufshcd_slave_alloc - handle initial SCSI device configurations
3322 * @sdev: pointer to SCSI device
3323 *
3324 * Returns success
3325 */
3326static int ufshcd_slave_alloc(struct scsi_device *sdev)
3327{
3328        struct ufs_hba *hba;
3329
3330        hba = shost_priv(sdev->host);
3331
3332        /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
3333        sdev->use_10_for_ms = 1;
3334
3335        /* allow SCSI layer to restart the device in case of errors */
3336        sdev->allow_restart = 1;
3337
3338        /* REPORT SUPPORTED OPERATION CODES is not supported */
3339        sdev->no_report_opcodes = 1;
3340
3341
3342        ufshcd_set_queue_depth(sdev);
3343
3344        ufshcd_get_lu_power_on_wp_status(hba, sdev);
3345
3346        return 0;
3347}
3348
3349/**
3350 * ufshcd_change_queue_depth - change queue depth
3351 * @sdev: pointer to SCSI device
3352 * @depth: required depth to set
3353 *
3354 * Change queue depth and make sure the max. limits are not crossed.
3355 */
3356static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
3357{
3358        struct ufs_hba *hba = shost_priv(sdev->host);
3359
3360        if (depth > hba->nutrs)
3361                depth = hba->nutrs;
3362        return scsi_change_queue_depth(sdev, depth);
3363}
3364
3365/**
3366 * ufshcd_slave_configure - adjust SCSI device configurations
3367 * @sdev: pointer to SCSI device
3368 */
3369static int ufshcd_slave_configure(struct scsi_device *sdev)
3370{
3371        struct request_queue *q = sdev->request_queue;
3372
3373        blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
3374        blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX);
3375
3376        return 0;
3377}
3378
3379/**
3380 * ufshcd_slave_destroy - remove SCSI device configurations
3381 * @sdev: pointer to SCSI device
3382 */
3383static void ufshcd_slave_destroy(struct scsi_device *sdev)
3384{
3385        struct ufs_hba *hba;
3386
3387        hba = shost_priv(sdev->host);
3388        /* Drop the reference as it won't be needed anymore */
3389        if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
3390                unsigned long flags;
3391
3392                spin_lock_irqsave(hba->host->host_lock, flags);
3393                hba->sdev_ufs_device = NULL;
3394                spin_unlock_irqrestore(hba->host->host_lock, flags);
3395        }
3396}
3397
3398/**
3399 * ufshcd_task_req_compl - handle task management request completion
3400 * @hba: per adapter instance
3401 * @index: index of the completed request
3402 * @resp: task management service response
3403 *
3404 * Returns non-zero value on error, zero on success
3405 */
3406static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
3407{
3408        struct utp_task_req_desc *task_req_descp;
3409        struct utp_upiu_task_rsp *task_rsp_upiup;
3410        unsigned long flags;
3411        int ocs_value;
3412        int task_result;
3413
3414        spin_lock_irqsave(hba->host->host_lock, flags);
3415
3416        /* Clear completed tasks from outstanding_tasks */
3417        __clear_bit(index, &hba->outstanding_tasks);
3418
3419        task_req_descp = hba->utmrdl_base_addr;
3420        ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
3421
3422        if (ocs_value == OCS_SUCCESS) {
3423                task_rsp_upiup = (struct utp_upiu_task_rsp *)
3424                                task_req_descp[index].task_rsp_upiu;
3425                task_result = be32_to_cpu(task_rsp_upiup->output_param1);
3426                task_result = task_result & MASK_TM_SERVICE_RESP;
3427                if (resp)
3428                        *resp = (u8)task_result;
3429        } else {
3430                dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
3431                                __func__, ocs_value);
3432        }
3433        spin_unlock_irqrestore(hba->host->host_lock, flags);
3434
3435        return ocs_value;
3436}
3437
3438/**
3439 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
3440 * @lrb: pointer to local reference block of completed command
3441 * @scsi_status: SCSI command status
3442 *
3443 * Returns value base on SCSI command status
3444 */
3445static inline int
3446ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
3447{
3448        int result = 0;
3449
3450        switch (scsi_status) {
3451        case SAM_STAT_CHECK_CONDITION:
3452                ufshcd_copy_sense_data(lrbp);
3453        case SAM_STAT_GOOD:
3454                result |= DID_OK << 16 |
3455                          COMMAND_COMPLETE << 8 |
3456                          scsi_status;
3457                break;
3458        case SAM_STAT_TASK_SET_FULL:
3459        case SAM_STAT_BUSY:
3460        case SAM_STAT_TASK_ABORTED:
3461                ufshcd_copy_sense_data(lrbp);
3462                result |= scsi_status;
3463                break;
3464        default:
3465                result |= DID_ERROR << 16;
3466                break;
3467        } /* end of switch */
3468
3469        return result;
3470}
3471
3472/**
3473 * ufshcd_transfer_rsp_status - Get overall status of the response
3474 * @hba: per adapter instance
3475 * @lrb: pointer to local reference block of completed command
3476 *
3477 * Returns result of the command to notify SCSI midlayer
3478 */
3479static inline int
3480ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3481{
3482        int result = 0;
3483        int scsi_status;
3484        int ocs;
3485
3486        /* overall command status of utrd */
3487        ocs = ufshcd_get_tr_ocs(lrbp);
3488
3489        switch (ocs) {
3490        case OCS_SUCCESS:
3491                result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3492
3493                switch (result) {
3494                case UPIU_TRANSACTION_RESPONSE:
3495                        /*
3496                         * get the response UPIU result to extract
3497                         * the SCSI command status
3498                         */
3499                        result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
3500
3501                        /*
3502                         * get the result based on SCSI status response
3503                         * to notify the SCSI midlayer of the command status
3504                         */
3505                        scsi_status = result & MASK_SCSI_STATUS;
3506                        result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
3507
3508                        /*
3509                         * Currently we are only supporting BKOPs exception
3510                         * events hence we can ignore BKOPs exception event
3511                         * during power management callbacks. BKOPs exception
3512                         * event is not expected to be raised in runtime suspend
3513                         * callback as it allows the urgent bkops.
3514                         * During system suspend, we are anyway forcefully
3515                         * disabling the bkops and if urgent bkops is needed
3516                         * it will be enabled on system resume. Long term
3517                         * solution could be to abort the system suspend if
3518                         * UFS device needs urgent BKOPs.
3519                         */
3520                        if (!hba->pm_op_in_progress &&
3521                            ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
3522                                schedule_work(&hba->eeh_work);
3523                        break;
3524                case UPIU_TRANSACTION_REJECT_UPIU:
3525                        /* TODO: handle Reject UPIU Response */
3526                        result = DID_ERROR << 16;
3527                        dev_err(hba->dev,
3528                                "Reject UPIU not fully implemented\n");
3529                        break;
3530                default:
3531                        result = DID_ERROR << 16;
3532                        dev_err(hba->dev,
3533                                "Unexpected request response code = %x\n",
3534                                result);
3535                        break;
3536                }
3537                break;
3538        case OCS_ABORTED:
3539                result |= DID_ABORT << 16;
3540                break;
3541        case OCS_INVALID_COMMAND_STATUS:
3542                result |= DID_REQUEUE << 16;
3543                break;
3544        case OCS_INVALID_CMD_TABLE_ATTR:
3545        case OCS_INVALID_PRDT_ATTR:
3546        case OCS_MISMATCH_DATA_BUF_SIZE:
3547        case OCS_MISMATCH_RESP_UPIU_SIZE:
3548        case OCS_PEER_COMM_FAILURE:
3549        case OCS_FATAL_ERROR:
3550        default:
3551                result |= DID_ERROR << 16;
3552                dev_err(hba->dev,
3553                "OCS error from controller = %x\n", ocs);
3554                break;
3555        } /* end of switch */
3556
3557        return result;
3558}
3559
3560/**
3561 * ufshcd_uic_cmd_compl - handle completion of uic command
3562 * @hba: per adapter instance
3563 * @intr_status: interrupt status generated by the controller
3564 */
3565static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
3566{
3567        if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
3568                hba->active_uic_cmd->argument2 |=
3569                        ufshcd_get_uic_cmd_result(hba);
3570                hba->active_uic_cmd->argument3 =
3571                        ufshcd_get_dme_attr_val(hba);
3572                complete(&hba->active_uic_cmd->done);
3573        }
3574
3575        if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done)
3576                complete(hba->uic_async_done);
3577}
3578
3579/**
3580 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
3581 * @hba: per adapter instance
3582 * @completed_reqs: requests to complete
3583 */
3584static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
3585                                        unsigned long completed_reqs)
3586{
3587        struct ufshcd_lrb *lrbp;
3588        struct scsi_cmnd *cmd;
3589        int result;
3590        int index;
3591
3592        for_each_set_bit(index, &completed_reqs, hba->nutrs) {
3593                lrbp = &hba->lrb[index];
3594                cmd = lrbp->cmd;
3595                if (cmd) {
3596                        result = ufshcd_transfer_rsp_status(hba, lrbp);
3597                        scsi_dma_unmap(cmd);
3598                        cmd->result = result;
3599                        /* Mark completed command as NULL in LRB */
3600                        lrbp->cmd = NULL;
3601                        clear_bit_unlock(index, &hba->lrb_in_use);
3602                        /* Do not touch lrbp after scsi done */
3603                        cmd->scsi_done(cmd);
3604                        __ufshcd_release(hba);
3605                } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
3606                        lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
3607                        if (hba->dev_cmd.complete)
3608                                complete(hba->dev_cmd.complete);
3609                }
3610        }
3611
3612        /* clear corresponding bits of completed commands */
3613        hba->outstanding_reqs ^= completed_reqs;
3614
3615        ufshcd_clk_scaling_update_busy(hba);
3616
3617        /* we might have free'd some tags above */
3618        wake_up(&hba->dev_cmd.tag_wq);
3619}
3620
3621/**
3622 * ufshcd_transfer_req_compl - handle SCSI and query command completion
3623 * @hba: per adapter instance
3624 */
3625static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
3626{
3627        unsigned long completed_reqs;
3628        u32 tr_doorbell;
3629
3630        /* Resetting interrupt aggregation counters first and reading the
3631         * DOOR_BELL afterward allows us to handle all the completed requests.
3632         * In order to prevent other interrupts starvation the DB is read once
3633         * after reset. The down side of this solution is the possibility of
3634         * false interrupt if device completes another request after resetting
3635         * aggregation and before reading the DB.
3636         */
3637        if (ufshcd_is_intr_aggr_allowed(hba))
3638                ufshcd_reset_intr_aggr(hba);
3639
3640        tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
3641        completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
3642
3643        __ufshcd_transfer_req_compl(hba, completed_reqs);
3644}
3645
3646/**
3647 * ufshcd_disable_ee - disable exception event
3648 * @hba: per-adapter instance
3649 * @mask: exception event to disable
3650 *
3651 * Disables exception event in the device so that the EVENT_ALERT
3652 * bit is not set.
3653 *
3654 * Returns zero on success, non-zero error value on failure.
3655 */
3656static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
3657{
3658        int err = 0;
3659        u32 val;
3660
3661        if (!(hba->ee_ctrl_mask & mask))
3662                goto out;
3663
3664        val = hba->ee_ctrl_mask & ~mask;
3665        val &= 0xFFFF; /* 2 bytes */
3666        err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3667                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3668        if (!err)
3669                hba->ee_ctrl_mask &= ~mask;
3670out:
3671        return err;
3672}
3673
3674/**
3675 * ufshcd_enable_ee - enable exception event
3676 * @hba: per-adapter instance
3677 * @mask: exception event to enable
3678 *
3679 * Enable corresponding exception event in the device to allow
3680 * device to alert host in critical scenarios.
3681 *
3682 * Returns zero on success, non-zero error value on failure.
3683 */
3684static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
3685{
3686        int err = 0;
3687        u32 val;
3688
3689        if (hba->ee_ctrl_mask & mask)
3690                goto out;
3691
3692        val = hba->ee_ctrl_mask | mask;
3693        val &= 0xFFFF; /* 2 bytes */
3694        err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3695                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3696        if (!err)
3697                hba->ee_ctrl_mask |= mask;
3698out:
3699        return err;
3700}
3701
3702/**
3703 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
3704 * @hba: per-adapter instance
3705 *
3706 * Allow device to manage background operations on its own. Enabling
3707 * this might lead to inconsistent latencies during normal data transfers
3708 * as the device is allowed to manage its own way of handling background
3709 * operations.
3710 *
3711 * Returns zero on success, non-zero on failure.
3712 */
3713static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
3714{
3715        int err = 0;
3716
3717        if (hba->auto_bkops_enabled)
3718                goto out;
3719
3720        err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
3721                        QUERY_FLAG_IDN_BKOPS_EN, NULL);
3722        if (err) {
3723                dev_err(hba->dev, "%s: failed to enable bkops %d\n",
3724                                __func__, err);
3725                goto out;
3726        }
3727
3728        hba->auto_bkops_enabled = true;
3729
3730        /* No need of URGENT_BKOPS exception from the device */
3731        err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3732        if (err)
3733                dev_err(hba->dev, "%s: failed to disable exception event %d\n",
3734                                __func__, err);
3735out:
3736        return err;
3737}
3738
3739/**
3740 * ufshcd_disable_auto_bkops - block device in doing background operations
3741 * @hba: per-adapter instance
3742 *
3743 * Disabling background operations improves command response latency but
3744 * has drawback of device moving into critical state where the device is
3745 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
3746 * host is idle so that BKOPS are managed effectively without any negative
3747 * impacts.
3748 *
3749 * Returns zero on success, non-zero on failure.
3750 */
3751static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
3752{
3753        int err = 0;
3754
3755        if (!hba->auto_bkops_enabled)
3756                goto out;
3757
3758        /*
3759         * If host assisted BKOPs is to be enabled, make sure
3760         * urgent bkops exception is allowed.
3761         */
3762        err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
3763        if (err) {
3764                dev_err(hba->dev, "%s: failed to enable exception event %d\n",
3765                                __func__, err);
3766                goto out;
3767        }
3768
3769        err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
3770                        QUERY_FLAG_IDN_BKOPS_EN, NULL);
3771        if (err) {
3772                dev_err(hba->dev, "%s: failed to disable bkops %d\n",
3773                                __func__, err);
3774                ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3775                goto out;
3776        }
3777
3778        hba->auto_bkops_enabled = false;
3779out:
3780        return err;
3781}
3782
3783/**
3784 * ufshcd_force_reset_auto_bkops - force enable of auto bkops
3785 * @hba: per adapter instance
3786 *
3787 * After a device reset the device may toggle the BKOPS_EN flag
3788 * to default value. The s/w tracking variables should be updated
3789 * as well. Do this by forcing enable of auto bkops.
3790 */
3791static void  ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
3792{
3793        hba->auto_bkops_enabled = false;
3794        hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
3795        ufshcd_enable_auto_bkops(hba);
3796}
3797
3798static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
3799{
3800        return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3801                        QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
3802}
3803
3804/**
3805 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
3806 * @hba: per-adapter instance
3807 * @status: bkops_status value
3808 *
3809 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
3810 * flag in the device to permit background operations if the device
3811 * bkops_status is greater than or equal to "status" argument passed to
3812 * this function, disable otherwise.
3813 *
3814 * Returns 0 for success, non-zero in case of failure.
3815 *
3816 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
3817 * to know whether auto bkops is enabled or disabled after this function
3818 * returns control to it.
3819 */
3820static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
3821                             enum bkops_status status)
3822{
3823        int err;
3824        u32 curr_status = 0;
3825
3826        err = ufshcd_get_bkops_status(hba, &curr_status);
3827        if (err) {
3828                dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
3829                                __func__, err);
3830                goto out;
3831        } else if (curr_status > BKOPS_STATUS_MAX) {
3832                dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
3833                                __func__, curr_status);
3834                err = -EINVAL;
3835                goto out;
3836        }
3837
3838        if (curr_status >= status)
3839                err = ufshcd_enable_auto_bkops(hba);
3840        else
3841                err = ufshcd_disable_auto_bkops(hba);
3842out:
3843        return err;
3844}
3845
3846/**
3847 * ufshcd_urgent_bkops - handle urgent bkops exception event
3848 * @hba: per-adapter instance
3849 *
3850 * Enable fBackgroundOpsEn flag in the device to permit background
3851 * operations.
3852 *
3853 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
3854 * and negative error value for any other failure.
3855 */
3856static int ufshcd_urgent_bkops(struct ufs_hba *hba)
3857{
3858        return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
3859}
3860
3861static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
3862{
3863        return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3864                        QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
3865}
3866
3867static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
3868{
3869        int err;
3870        u32 curr_status = 0;
3871
3872        if (hba->is_urgent_bkops_lvl_checked)
3873                goto enable_auto_bkops;
3874
3875        err = ufshcd_get_bkops_status(hba, &curr_status);
3876        if (err) {
3877                dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
3878                                __func__, err);
3879                goto out;
3880        }
3881
3882        /*
3883         * We are seeing that some devices are raising the urgent bkops
3884         * exception events even when BKOPS status doesn't indicate performace
3885         * impacted or critical. Handle these device by determining their urgent
3886         * bkops status at runtime.
3887         */
3888        if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
3889                dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
3890                                __func__, curr_status);
3891                /* update the current status as the urgent bkops level */
3892                hba->urgent_bkops_lvl = curr_status;
3893                hba->is_urgent_bkops_lvl_checked = true;
3894        }
3895
3896enable_auto_bkops:
3897        err = ufshcd_enable_auto_bkops(hba);
3898out:
3899        if (err < 0)
3900                dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
3901                                __func__, err);
3902}
3903
3904/**
3905 * ufshcd_exception_event_handler - handle exceptions raised by device
3906 * @work: pointer to work data
3907 *
3908 * Read bExceptionEventStatus attribute from the device and handle the
3909 * exception event accordingly.
3910 */
3911static void ufshcd_exception_event_handler(struct work_struct *work)
3912{
3913        struct ufs_hba *hba;
3914        int err;
3915        u32 status = 0;
3916        hba = container_of(work, struct ufs_hba, eeh_work);
3917
3918        pm_runtime_get_sync(hba->dev);
3919        err = ufshcd_get_ee_status(hba, &status);
3920        if (err) {
3921                dev_err(hba->dev, "%s: failed to get exception status %d\n",
3922                                __func__, err);
3923                goto out;
3924        }
3925
3926        status &= hba->ee_ctrl_mask;
3927
3928        if (status & MASK_EE_URGENT_BKOPS)
3929                ufshcd_bkops_exception_event_handler(hba);
3930
3931out:
3932        pm_runtime_put_sync(hba->dev);
3933        return;
3934}
3935
3936/* Complete requests that have door-bell cleared */
3937static void ufshcd_complete_requests(struct ufs_hba *hba)
3938{
3939        ufshcd_transfer_req_compl(hba);
3940        ufshcd_tmc_handler(hba);
3941}
3942
3943/**
3944 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
3945 *                              to recover from the DL NAC errors or not.
3946 * @hba: per-adapter instance
3947 *
3948 * Returns true if error handling is required, false otherwise
3949 */
3950static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
3951{
3952        unsigned long flags;
3953        bool err_handling = true;
3954
3955        spin_lock_irqsave(hba->host->host_lock, flags);
3956        /*
3957         * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
3958         * device fatal error and/or DL NAC & REPLAY timeout errors.
3959         */
3960        if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
3961                goto out;
3962
3963        if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
3964            ((hba->saved_err & UIC_ERROR) &&
3965             (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
3966                goto out;
3967
3968        if ((hba->saved_err & UIC_ERROR) &&
3969            (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
3970                int err;
3971                /*
3972                 * wait for 50ms to see if we can get any other errors or not.
3973                 */
3974                spin_unlock_irqrestore(hba->host->host_lock, flags);
3975                msleep(50);
3976                spin_lock_irqsave(hba->host->host_lock, flags);
3977
3978                /*
3979                 * now check if we have got any other severe errors other than
3980                 * DL NAC error?
3981                 */
3982                if ((hba->saved_err & INT_FATAL_ERRORS) ||
3983                    ((hba->saved_err & UIC_ERROR) &&
3984                    (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
3985                        goto out;
3986
3987                /*
3988                 * As DL NAC is the only error received so far, send out NOP
3989                 * command to confirm if link is still active or not.
3990                 *   - If we don't get any response then do error recovery.
3991                 *   - If we get response then clear the DL NAC error bit.
3992                 */
3993
3994                spin_unlock_irqrestore(hba->host->host_lock, flags);
3995                err = ufshcd_verify_dev_init(hba);
3996                spin_lock_irqsave(hba->host->host_lock, flags);
3997
3998                if (err)
3999                        goto out;
4000
4001                /* Link seems to be alive hence ignore the DL NAC errors */
4002                if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
4003                        hba->saved_err &= ~UIC_ERROR;
4004                /* clear NAC error */
4005                hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
4006                if (!hba->saved_uic_err) {
4007                        err_handling = false;
4008                        goto out;
4009                }
4010        }
4011out:
4012        spin_unlock_irqrestore(hba->host->host_lock, flags);
4013        return err_handling;
4014}
4015
4016/**
4017 * ufshcd_err_handler - handle UFS errors that require s/w attention
4018 * @work: pointer to work structure
4019 */
4020static void ufshcd_err_handler(struct work_struct *work)
4021{
4022        struct ufs_hba *hba;
4023        unsigned long flags;
4024        u32 err_xfer = 0;
4025        u32 err_tm = 0;
4026        int err = 0;
4027        int tag;
4028        bool needs_reset = false;
4029
4030        hba = container_of(work, struct ufs_hba, eh_work);
4031
4032        pm_runtime_get_sync(hba->dev);
4033        ufshcd_hold(hba, false);
4034
4035        spin_lock_irqsave(hba->host->host_lock, flags);
4036        if (hba->ufshcd_state == UFSHCD_STATE_RESET)
4037                goto out;
4038
4039        hba->ufshcd_state = UFSHCD_STATE_RESET;
4040        ufshcd_set_eh_in_progress(hba);
4041
4042        /* Complete requests that have door-bell cleared by h/w */
4043        ufshcd_complete_requests(hba);
4044
4045        if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
4046                bool ret;
4047
4048                spin_unlock_irqrestore(hba->host->host_lock, flags);
4049                /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
4050                ret = ufshcd_quirk_dl_nac_errors(hba);
4051                spin_lock_irqsave(hba->host->host_lock, flags);
4052                if (!ret)
4053                        goto skip_err_handling;
4054        }
4055        if ((hba->saved_err & INT_FATAL_ERRORS) ||
4056            ((hba->saved_err & UIC_ERROR) &&
4057            (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
4058                                   UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
4059                                   UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
4060                needs_reset = true;
4061
4062        /*
4063         * if host reset is required then skip clearing the pending
4064         * transfers forcefully because they will automatically get
4065         * cleared after link startup.
4066         */
4067        if (needs_reset)
4068                goto skip_pending_xfer_clear;
4069
4070        /* release lock as clear command might sleep */
4071        spin_unlock_irqrestore(hba->host->host_lock, flags);
4072        /* Clear pending transfer requests */
4073        for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
4074                if (ufshcd_clear_cmd(hba, tag)) {
4075                        err_xfer = true;
4076                        goto lock_skip_pending_xfer_clear;
4077                }
4078        }
4079
4080        /* Clear pending task management requests */
4081        for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
4082                if (ufshcd_clear_tm_cmd(hba, tag)) {
4083                        err_tm = true;
4084                        goto lock_skip_pending_xfer_clear;
4085                }
4086        }
4087
4088lock_skip_pending_xfer_clear:
4089        spin_lock_irqsave(hba->host->host_lock, flags);
4090
4091        /* Complete the requests that are cleared by s/w */
4092        ufshcd_complete_requests(hba);
4093
4094        if (err_xfer || err_tm)
4095                needs_reset = true;
4096
4097skip_pending_xfer_clear:
4098        /* Fatal errors need reset */
4099        if (needs_reset) {
4100                unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
4101
4102                /*
4103                 * ufshcd_reset_and_restore() does the link reinitialization
4104                 * which will need atleast one empty doorbell slot to send the
4105                 * device management commands (NOP and query commands).
4106                 * If there is no slot empty at this moment then free up last
4107                 * slot forcefully.
4108                 */
4109                if (hba->outstanding_reqs == max_doorbells)
4110                        __ufshcd_transfer_req_compl(hba,
4111                                                    (1UL << (hba->nutrs - 1)));
4112
4113                spin_unlock_irqrestore(hba->host->host_lock, flags);
4114                err = ufshcd_reset_and_restore(hba);
4115                spin_lock_irqsave(hba->host->host_lock, flags);
4116                if (err) {
4117                        dev_err(hba->dev, "%s: reset and restore failed\n",
4118                                        __func__);
4119                        hba->ufshcd_state = UFSHCD_STATE_ERROR;
4120                }
4121                /*
4122                 * Inform scsi mid-layer that we did reset and allow to handle
4123                 * Unit Attention properly.
4124                 */
4125                scsi_report_bus_reset(hba->host, 0);
4126                hba->saved_err = 0;
4127                hba->saved_uic_err = 0;
4128        }
4129
4130skip_err_handling:
4131        if (!needs_reset) {
4132                hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4133                if (hba->saved_err || hba->saved_uic_err)
4134                        dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
4135                            __func__, hba->saved_err, hba->saved_uic_err);
4136        }
4137
4138        ufshcd_clear_eh_in_progress(hba);
4139
4140out:
4141        spin_unlock_irqrestore(hba->host->host_lock, flags);
4142        scsi_unblock_requests(hba->host);
4143        ufshcd_release(hba);
4144        pm_runtime_put_sync(hba->dev);
4145}
4146
4147/**
4148 * ufshcd_update_uic_error - check and set fatal UIC error flags.
4149 * @hba: per-adapter instance
4150 */
4151static void ufshcd_update_uic_error(struct ufs_hba *hba)
4152{
4153        u32 reg;
4154
4155        /* PA_INIT_ERROR is fatal and needs UIC reset */
4156        reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
4157        if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
4158                hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
4159        else if (hba->dev_quirks &
4160                   UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
4161                if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
4162                        hba->uic_error |=
4163                                UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
4164                else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
4165                        hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
4166        }
4167
4168        /* UIC NL/TL/DME errors needs software retry */
4169        reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
4170        if (reg)
4171                hba->uic_error |= UFSHCD_UIC_NL_ERROR;
4172
4173        reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
4174        if (reg)
4175                hba->uic_error |= UFSHCD_UIC_TL_ERROR;
4176
4177        reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
4178        if (reg)
4179                hba->uic_error |= UFSHCD_UIC_DME_ERROR;
4180
4181        dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
4182                        __func__, hba->uic_error);
4183}
4184
4185/**
4186 * ufshcd_check_errors - Check for errors that need s/w attention
4187 * @hba: per-adapter instance
4188 */
4189static void ufshcd_check_errors(struct ufs_hba *hba)
4190{
4191        bool queue_eh_work = false;
4192
4193        if (hba->errors & INT_FATAL_ERRORS)
4194                queue_eh_work = true;
4195
4196        if (hba->errors & UIC_ERROR) {
4197                hba->uic_error = 0;
4198                ufshcd_update_uic_error(hba);
4199                if (hba->uic_error)
4200                        queue_eh_work = true;
4201        }
4202
4203        if (queue_eh_work) {
4204                /*
4205                 * update the transfer error masks to sticky bits, let's do this
4206                 * irrespective of current ufshcd_state.
4207                 */
4208                hba->saved_err |= hba->errors;
4209                hba->saved_uic_err |= hba->uic_error;
4210
4211                /* handle fatal errors only when link is functional */
4212                if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
4213                        /* block commands from scsi mid-layer */
4214                        scsi_block_requests(hba->host);
4215
4216                        hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
4217                        schedule_work(&hba->eh_work);
4218                }
4219        }
4220        /*
4221         * if (!queue_eh_work) -
4222         * Other errors are either non-fatal where host recovers
4223         * itself without s/w intervention or errors that will be
4224         * handled by the SCSI core layer.
4225         */
4226}
4227
4228/**
4229 * ufshcd_tmc_handler - handle task management function completion
4230 * @hba: per adapter instance
4231 */
4232static void ufshcd_tmc_handler(struct ufs_hba *hba)
4233{
4234        u32 tm_doorbell;
4235
4236        tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
4237        hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
4238        wake_up(&hba->tm_wq);
4239}
4240
4241/**
4242 * ufshcd_sl_intr - Interrupt service routine
4243 * @hba: per adapter instance
4244 * @intr_status: contains interrupts generated by the controller
4245 */
4246static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
4247{
4248        hba->errors = UFSHCD_ERROR_MASK & intr_status;
4249        if (hba->errors)
4250                ufshcd_check_errors(hba);
4251
4252        if (intr_status & UFSHCD_UIC_MASK)
4253                ufshcd_uic_cmd_compl(hba, intr_status);
4254
4255        if (intr_status & UTP_TASK_REQ_COMPL)
4256                ufshcd_tmc_handler(hba);
4257
4258        if (intr_status & UTP_TRANSFER_REQ_COMPL)
4259                ufshcd_transfer_req_compl(hba);
4260}
4261
4262/**
4263 * ufshcd_intr - Main interrupt service routine
4264 * @irq: irq number
4265 * @__hba: pointer to adapter instance
4266 *
4267 * Returns IRQ_HANDLED - If interrupt is valid
4268 *              IRQ_NONE - If invalid interrupt
4269 */
4270static irqreturn_t ufshcd_intr(int irq, void *__hba)
4271{
4272        u32 intr_status, enabled_intr_status;
4273        irqreturn_t retval = IRQ_NONE;
4274        struct ufs_hba *hba = __hba;
4275
4276        spin_lock(hba->host->host_lock);
4277        intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
4278        enabled_intr_status =
4279                intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4280
4281        if (intr_status)
4282                ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
4283
4284        if (enabled_intr_status) {
4285                ufshcd_sl_intr(hba, enabled_intr_status);
4286                retval = IRQ_HANDLED;
4287        }
4288        spin_unlock(hba->host->host_lock);
4289        return retval;
4290}
4291
4292static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
4293{
4294        int err = 0;
4295        u32 mask = 1 << tag;
4296        unsigned long flags;
4297
4298        if (!test_bit(tag, &hba->outstanding_tasks))
4299                goto out;
4300
4301        spin_lock_irqsave(hba->host->host_lock, flags);
4302        ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
4303        spin_unlock_irqrestore(hba->host->host_lock, flags);
4304
4305        /* poll for max. 1 sec to clear door bell register by h/w */
4306        err = ufshcd_wait_for_register(hba,
4307                        REG_UTP_TASK_REQ_DOOR_BELL,
4308                        mask, 0, 1000, 1000, true);
4309out:
4310        return err;
4311}
4312
4313/**
4314 * ufshcd_issue_tm_cmd - issues task management commands to controller
4315 * @hba: per adapter instance
4316 * @lun_id: LUN ID to which TM command is sent
4317 * @task_id: task ID to which the TM command is applicable
4318 * @tm_function: task management function opcode
4319 * @tm_response: task management service response return value
4320 *
4321 * Returns non-zero value on error, zero on success.
4322 */
4323static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
4324                u8 tm_function, u8 *tm_response)
4325{
4326        struct utp_task_req_desc *task_req_descp;
4327        struct utp_upiu_task_req *task_req_upiup;
4328        struct Scsi_Host *host;
4329        unsigned long flags;
4330        int free_slot;
4331        int err;
4332        int task_tag;
4333
4334        host = hba->host;
4335
4336        /*
4337         * Get free slot, sleep if slots are unavailable.
4338         * Even though we use wait_event() which sleeps indefinitely,
4339         * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
4340         */
4341        wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot));
4342        ufshcd_hold(hba, false);
4343
4344        spin_lock_irqsave(host->host_lock, flags);
4345        task_req_descp = hba->utmrdl_base_addr;
4346        task_req_descp += free_slot;
4347
4348        /* Configure task request descriptor */
4349        task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
4350        task_req_descp->header.dword_2 =
4351                        cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
4352
4353        /* Configure task request UPIU */
4354        task_req_upiup =
4355                (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
4356        task_tag = hba->nutrs + free_slot;
4357        task_req_upiup->header.dword_0 =
4358                UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
4359                                              lun_id, task_tag);
4360        task_req_upiup->header.dword_1 =
4361                UPIU_HEADER_DWORD(0, tm_function, 0, 0);
4362        /*
4363         * The host shall provide the same value for LUN field in the basic
4364         * header and for Input Parameter.
4365         */
4366        task_req_upiup->input_param1 = cpu_to_be32(lun_id);
4367        task_req_upiup->input_param2 = cpu_to_be32(task_id);
4368
4369        /* send command to the controller */
4370        __set_bit(free_slot, &hba->outstanding_tasks);
4371
4372        /* Make sure descriptors are ready before ringing the task doorbell */
4373        wmb();
4374
4375        ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
4376
4377        spin_unlock_irqrestore(host->host_lock, flags);
4378
4379        /* wait until the task management command is completed */
4380        err = wait_event_timeout(hba->tm_wq,
4381                        test_bit(free_slot, &hba->tm_condition),
4382                        msecs_to_jiffies(TM_CMD_TIMEOUT));
4383        if (!err) {
4384                dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
4385                                __func__, tm_function);
4386                if (ufshcd_clear_tm_cmd(hba, free_slot))
4387                        dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
4388                                        __func__, free_slot);
4389                err = -ETIMEDOUT;
4390        } else {
4391                err = ufshcd_task_req_compl(hba, free_slot, tm_response);
4392        }
4393
4394        clear_bit(free_slot, &hba->tm_condition);
4395        ufshcd_put_tm_slot(hba, free_slot);
4396        wake_up(&hba->tm_tag_wq);
4397
4398        ufshcd_release(hba);
4399        return err;
4400}
4401
4402/**
4403 * ufshcd_eh_device_reset_handler - device reset handler registered to
4404 *                                    scsi layer.
4405 * @cmd: SCSI command pointer
4406 *
4407 * Returns SUCCESS/FAILED
4408 */
4409static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
4410{
4411        struct Scsi_Host *host;
4412        struct ufs_hba *hba;
4413        unsigned int tag;
4414        u32 pos;
4415        int err;
4416        u8 resp = 0xF;
4417        struct ufshcd_lrb *lrbp;
4418        unsigned long flags;
4419
4420        host = cmd->device->host;
4421        hba = shost_priv(host);
4422        tag = cmd->request->tag;
4423
4424        lrbp = &hba->lrb[tag];
4425        err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
4426        if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4427                if (!err)
4428                        err = resp;
4429                goto out;
4430        }
4431
4432        /* clear the commands that were pending for corresponding LUN */
4433        for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
4434                if (hba->lrb[pos].lun == lrbp->lun) {
4435                        err = ufshcd_clear_cmd(hba, pos);
4436                        if (err)
4437                                break;
4438                }
4439        }
4440        spin_lock_irqsave(host->host_lock, flags);
4441        ufshcd_transfer_req_compl(hba);
4442        spin_unlock_irqrestore(host->host_lock, flags);
4443out:
4444        if (!err) {
4445                err = SUCCESS;
4446        } else {
4447                dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4448                err = FAILED;
4449        }
4450        return err;
4451}
4452
4453/**
4454 * ufshcd_abort - abort a specific command
4455 * @cmd: SCSI command pointer
4456 *
4457 * Abort the pending command in device by sending UFS_ABORT_TASK task management
4458 * command, and in host controller by clearing the door-bell register. There can
4459 * be race between controller sending the command to the device while abort is
4460 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
4461 * really issued and then try to abort it.
4462 *
4463 * Returns SUCCESS/FAILED
4464 */
4465static int ufshcd_abort(struct scsi_cmnd *cmd)
4466{
4467        struct Scsi_Host *host;
4468        struct ufs_hba *hba;
4469        unsigned long flags;
4470        unsigned int tag;
4471        int err = 0;
4472        int poll_cnt;
4473        u8 resp = 0xF;
4474        struct ufshcd_lrb *lrbp;
4475        u32 reg;
4476
4477        host = cmd->device->host;
4478        hba = shost_priv(host);
4479        tag = cmd->request->tag;
4480        if (!ufshcd_valid_tag(hba, tag)) {
4481                dev_err(hba->dev,
4482                        "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
4483                        __func__, tag, cmd, cmd->request);
4484                BUG();
4485        }
4486
4487        ufshcd_hold(hba, false);
4488        reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4489        /* If command is already aborted/completed, return SUCCESS */
4490        if (!(test_bit(tag, &hba->outstanding_reqs))) {
4491                dev_err(hba->dev,
4492                        "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
4493                        __func__, tag, hba->outstanding_reqs, reg);
4494                goto out;
4495        }
4496
4497        if (!(reg & (1 << tag))) {
4498                dev_err(hba->dev,
4499                "%s: cmd was completed, but without a notifying intr, tag = %d",
4500                __func__, tag);
4501        }
4502
4503        lrbp = &hba->lrb[tag];
4504        for (poll_cnt = 100; poll_cnt; poll_cnt--) {
4505                err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4506                                UFS_QUERY_TASK, &resp);
4507                if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
4508                        /* cmd pending in the device */
4509                        break;
4510                } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4511                        /*
4512                         * cmd not pending in the device, check if it is
4513                         * in transition.
4514                         */
4515                        reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4516                        if (reg & (1 << tag)) {
4517                                /* sleep for max. 200us to stabilize */
4518                                usleep_range(100, 200);
4519                                continue;
4520                        }
4521                        /* command completed already */
4522                        goto out;
4523                } else {
4524                        if (!err)
4525                                err = resp; /* service response error */
4526                        goto out;
4527                }
4528        }
4529
4530        if (!poll_cnt) {
4531                err = -EBUSY;
4532                goto out;
4533        }
4534
4535        err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4536                        UFS_ABORT_TASK, &resp);
4537        if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4538                if (!err)
4539                        err = resp; /* service response error */
4540                goto out;
4541        }
4542
4543        err = ufshcd_clear_cmd(hba, tag);
4544        if (err)
4545                goto out;
4546
4547        scsi_dma_unmap(cmd);
4548
4549        spin_lock_irqsave(host->host_lock, flags);
4550        ufshcd_outstanding_req_clear(hba, tag);
4551        hba->lrb[tag].cmd = NULL;
4552        spin_unlock_irqrestore(host->host_lock, flags);
4553
4554        clear_bit_unlock(tag, &hba->lrb_in_use);
4555        wake_up(&hba->dev_cmd.tag_wq);
4556
4557out:
4558        if (!err) {
4559                err = SUCCESS;
4560        } else {
4561                dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4562                err = FAILED;
4563        }
4564
4565        /*
4566         * This ufshcd_release() corresponds to the original scsi cmd that got
4567         * aborted here (as we won't get any IRQ for it).
4568         */
4569        ufshcd_release(hba);
4570        return err;
4571}
4572
4573/**
4574 * ufshcd_host_reset_and_restore - reset and restore host controller
4575 * @hba: per-adapter instance
4576 *
4577 * Note that host controller reset may issue DME_RESET to
4578 * local and remote (device) Uni-Pro stack and the attributes
4579 * are reset to default state.
4580 *
4581 * Returns zero on success, non-zero on failure
4582 */
4583static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
4584{
4585        int err;
4586        unsigned long flags;
4587
4588        /* Reset the host controller */
4589        spin_lock_irqsave(hba->host->host_lock, flags);
4590        ufshcd_hba_stop(hba, false);
4591        spin_unlock_irqrestore(hba->host->host_lock, flags);
4592
4593        err = ufshcd_hba_enable(hba);
4594        if (err)
4595                goto out;
4596
4597        /* Establish the link again and restore the device */
4598        err = ufshcd_probe_hba(hba);
4599
4600        if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
4601                err = -EIO;
4602out:
4603        if (err)
4604                dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
4605
4606        return err;
4607}
4608
4609/**
4610 * ufshcd_reset_and_restore - reset and re-initialize host/device
4611 * @hba: per-adapter instance
4612 *
4613 * Reset and recover device, host and re-establish link. This
4614 * is helpful to recover the communication in fatal error conditions.
4615 *
4616 * Returns zero on success, non-zero on failure
4617 */
4618static int ufshcd_reset_and_restore(struct ufs_hba *hba)
4619{
4620        int err = 0;
4621        unsigned long flags;
4622        int retries = MAX_HOST_RESET_RETRIES;
4623
4624        do {
4625                err = ufshcd_host_reset_and_restore(hba);
4626        } while (err && --retries);
4627
4628        /*
4629         * After reset the door-bell might be cleared, complete
4630         * outstanding requests in s/w here.
4631         */
4632        spin_lock_irqsave(hba->host->host_lock, flags);
4633        ufshcd_transfer_req_compl(hba);
4634        ufshcd_tmc_handler(hba);
4635        spin_unlock_irqrestore(hba->host->host_lock, flags);
4636
4637        return err;
4638}
4639
4640/**
4641 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
4642 * @cmd - SCSI command pointer
4643 *
4644 * Returns SUCCESS/FAILED
4645 */
4646static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
4647{
4648        int err;
4649        unsigned long flags;
4650        struct ufs_hba *hba;
4651
4652        hba = shost_priv(cmd->device->host);
4653
4654        ufshcd_hold(hba, false);
4655        /*
4656         * Check if there is any race with fatal error handling.
4657         * If so, wait for it to complete. Even though fatal error
4658         * handling does reset and restore in some cases, don't assume
4659         * anything out of it. We are just avoiding race here.
4660         */
4661        do {
4662                spin_lock_irqsave(hba->host->host_lock, flags);
4663                if (!(work_pending(&hba->eh_work) ||
4664                                hba->ufshcd_state == UFSHCD_STATE_RESET))
4665                        break;
4666                spin_unlock_irqrestore(hba->host->host_lock, flags);
4667                dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
4668                flush_work(&hba->eh_work);
4669        } while (1);
4670
4671        hba->ufshcd_state = UFSHCD_STATE_RESET;
4672        ufshcd_set_eh_in_progress(hba);
4673        spin_unlock_irqrestore(hba->host->host_lock, flags);
4674
4675        err = ufshcd_reset_and_restore(hba);
4676
4677        spin_lock_irqsave(hba->host->host_lock, flags);
4678        if (!err) {
4679                err = SUCCESS;
4680                hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4681        } else {
4682                err = FAILED;
4683                hba->ufshcd_state = UFSHCD_STATE_ERROR;
4684        }
4685        ufshcd_clear_eh_in_progress(hba);
4686        spin_unlock_irqrestore(hba->host->host_lock, flags);
4687
4688        ufshcd_release(hba);
4689        return err;
4690}
4691
4692/**
4693 * ufshcd_get_max_icc_level - calculate the ICC level
4694 * @sup_curr_uA: max. current supported by the regulator
4695 * @start_scan: row at the desc table to start scan from
4696 * @buff: power descriptor buffer
4697 *
4698 * Returns calculated max ICC level for specific regulator
4699 */
4700static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
4701{
4702        int i;
4703        int curr_uA;
4704        u16 data;
4705        u16 unit;
4706
4707        for (i = start_scan; i >= 0; i--) {
4708                data = be16_to_cpu(*((u16 *)(buff + 2*i)));
4709                unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
4710                                                ATTR_ICC_LVL_UNIT_OFFSET;
4711                curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
4712                switch (unit) {
4713                case UFSHCD_NANO_AMP:
4714                        curr_uA = curr_uA / 1000;
4715                        break;
4716                case UFSHCD_MILI_AMP:
4717                        curr_uA = curr_uA * 1000;
4718                        break;
4719                case UFSHCD_AMP:
4720                        curr_uA = curr_uA * 1000 * 1000;
4721                        break;
4722                case UFSHCD_MICRO_AMP:
4723                default:
4724                        break;
4725                }
4726                if (sup_curr_uA >= curr_uA)
4727                        break;
4728        }
4729        if (i < 0) {
4730                i = 0;
4731                pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
4732        }
4733
4734        return (u32)i;
4735}
4736
4737/**
4738 * ufshcd_calc_icc_level - calculate the max ICC level
4739 * In case regulators are not initialized we'll return 0
4740 * @hba: per-adapter instance
4741 * @desc_buf: power descriptor buffer to extract ICC levels from.
4742 * @len: length of desc_buff
4743 *
4744 * Returns calculated ICC level
4745 */
4746static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
4747                                                        u8 *desc_buf, int len)
4748{
4749        u32 icc_level = 0;
4750
4751        if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
4752                                                !hba->vreg_info.vccq2) {
4753                dev_err(hba->dev,
4754                        "%s: Regulator capability was not set, actvIccLevel=%d",
4755                                                        __func__, icc_level);
4756                goto out;
4757        }
4758
4759        if (hba->vreg_info.vcc)
4760                icc_level = ufshcd_get_max_icc_level(
4761                                hba->vreg_info.vcc->max_uA,
4762                                POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
4763                                &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
4764
4765        if (hba->vreg_info.vccq)
4766                icc_level = ufshcd_get_max_icc_level(
4767                                hba->vreg_info.vccq->max_uA,
4768                                icc_level,
4769                                &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
4770
4771        if (hba->vreg_info.vccq2)
4772                icc_level = ufshcd_get_max_icc_level(
4773                                hba->vreg_info.vccq2->max_uA,
4774                                icc_level,
4775                                &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
4776out:
4777        return icc_level;
4778}
4779
4780static void ufshcd_init_icc_levels(struct ufs_hba *hba)
4781{
4782        int ret;
4783        int buff_len = QUERY_DESC_POWER_MAX_SIZE;
4784        u8 desc_buf[QUERY_DESC_POWER_MAX_SIZE];
4785
4786        ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
4787        if (ret) {
4788                dev_err(hba->dev,
4789                        "%s: Failed reading power descriptor.len = %d ret = %d",
4790                        __func__, buff_len, ret);
4791                return;
4792        }
4793
4794        hba->init_prefetch_data.icc_level =
4795                        ufshcd_find_max_sup_active_icc_level(hba,
4796                        desc_buf, buff_len);
4797        dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
4798                        __func__, hba->init_prefetch_data.icc_level);
4799
4800        ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4801                QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
4802                &hba->init_prefetch_data.icc_level);
4803
4804        if (ret)
4805                dev_err(hba->dev,
4806                        "%s: Failed configuring bActiveICCLevel = %d ret = %d",
4807                        __func__, hba->init_prefetch_data.icc_level , ret);
4808
4809}
4810
4811/**
4812 * ufshcd_scsi_add_wlus - Adds required W-LUs
4813 * @hba: per-adapter instance
4814 *
4815 * UFS device specification requires the UFS devices to support 4 well known
4816 * logical units:
4817 *      "REPORT_LUNS" (address: 01h)
4818 *      "UFS Device" (address: 50h)
4819 *      "RPMB" (address: 44h)
4820 *      "BOOT" (address: 30h)
4821 * UFS device's power management needs to be controlled by "POWER CONDITION"
4822 * field of SSU (START STOP UNIT) command. But this "power condition" field
4823 * will take effect only when its sent to "UFS device" well known logical unit
4824 * hence we require the scsi_device instance to represent this logical unit in
4825 * order for the UFS host driver to send the SSU command for power management.
4826
4827 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
4828 * Block) LU so user space process can control this LU. User space may also
4829 * want to have access to BOOT LU.
4830
4831 * This function adds scsi device instances for each of all well known LUs
4832 * (except "REPORT LUNS" LU).
4833 *
4834 * Returns zero on success (all required W-LUs are added successfully),
4835 * non-zero error value on failure (if failed to add any of the required W-LU).
4836 */
4837static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
4838{
4839        int ret = 0;
4840        struct scsi_device *sdev_rpmb;
4841        struct scsi_device *sdev_boot;
4842
4843        hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
4844                ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
4845        if (IS_ERR(hba->sdev_ufs_device)) {
4846                ret = PTR_ERR(hba->sdev_ufs_device);
4847                hba->sdev_ufs_device = NULL;
4848                goto out;
4849        }
4850        scsi_device_put(hba->sdev_ufs_device);
4851
4852        sdev_boot = __scsi_add_device(hba->host, 0, 0,
4853                ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
4854        if (IS_ERR(sdev_boot)) {
4855                ret = PTR_ERR(sdev_boot);
4856                goto remove_sdev_ufs_device;
4857        }
4858        scsi_device_put(sdev_boot);
4859
4860        sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
4861                ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
4862        if (IS_ERR(sdev_rpmb)) {
4863                ret = PTR_ERR(sdev_rpmb);
4864                goto remove_sdev_boot;
4865        }
4866        scsi_device_put(sdev_rpmb);
4867        goto out;
4868
4869remove_sdev_boot:
4870        scsi_remove_device(sdev_boot);
4871remove_sdev_ufs_device:
4872        scsi_remove_device(hba->sdev_ufs_device);
4873out:
4874        return ret;
4875}
4876
4877static int ufs_get_device_info(struct ufs_hba *hba,
4878                                struct ufs_device_info *card_data)
4879{
4880        int err;
4881        u8 model_index;
4882        u8 str_desc_buf[QUERY_DESC_STRING_MAX_SIZE + 1] = {0};
4883        u8 desc_buf[QUERY_DESC_DEVICE_MAX_SIZE];
4884
4885        err = ufshcd_read_device_desc(hba, desc_buf,
4886                                        QUERY_DESC_DEVICE_MAX_SIZE);
4887        if (err) {
4888                dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
4889                        __func__, err);
4890                goto out;
4891        }
4892
4893        /*
4894         * getting vendor (manufacturerID) and Bank Index in big endian
4895         * format
4896         */
4897        card_data->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
4898                                     desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
4899
4900        model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4901
4902        err = ufshcd_read_string_desc(hba, model_index, str_desc_buf,
4903                                        QUERY_DESC_STRING_MAX_SIZE, ASCII_STD);
4904        if (err) {
4905                dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
4906                        __func__, err);
4907                goto out;
4908        }
4909
4910        str_desc_buf[QUERY_DESC_STRING_MAX_SIZE] = '\0';
4911        strlcpy(card_data->model, (str_desc_buf + QUERY_DESC_HDR_SIZE),
4912                min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET],
4913                      MAX_MODEL_LEN));
4914
4915        /* Null terminate the model string */
4916        card_data->model[MAX_MODEL_LEN] = '\0';
4917
4918out:
4919        return err;
4920}
4921
4922void ufs_advertise_fixup_device(struct ufs_hba *hba)
4923{
4924        int err;
4925        struct ufs_dev_fix *f;
4926        struct ufs_device_info card_data;
4927
4928        card_data.wmanufacturerid = 0;
4929
4930        err = ufs_get_device_info(hba, &card_data);
4931        if (err) {
4932                dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
4933                        __func__, err);
4934                return;
4935        }
4936
4937        for (f = ufs_fixups; f->quirk; f++) {
4938                if (((f->card.wmanufacturerid == card_data.wmanufacturerid) ||
4939                    (f->card.wmanufacturerid == UFS_ANY_VENDOR)) &&
4940                    (STR_PRFX_EQUAL(f->card.model, card_data.model) ||
4941                     !strcmp(f->card.model, UFS_ANY_MODEL)))
4942                        hba->dev_quirks |= f->quirk;
4943        }
4944}
4945
4946/**
4947 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
4948 * @hba: per-adapter instance
4949 *
4950 * PA_TActivate parameter can be tuned manually if UniPro version is less than
4951 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
4952 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
4953 * the hibern8 exit latency.
4954 *
4955 * Returns zero on success, non-zero error value on failure.
4956 */
4957static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
4958{
4959        int ret = 0;
4960        u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
4961
4962        ret = ufshcd_dme_peer_get(hba,
4963                                  UIC_ARG_MIB_SEL(
4964                                        RX_MIN_ACTIVATETIME_CAPABILITY,
4965                                        UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
4966                                  &peer_rx_min_activatetime);
4967        if (ret)
4968                goto out;
4969
4970        /* make sure proper unit conversion is applied */
4971        tuned_pa_tactivate =
4972                ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
4973                 / PA_TACTIVATE_TIME_UNIT_US);
4974        ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
4975                             tuned_pa_tactivate);
4976
4977out:
4978        return ret;
4979}
4980
4981/**
4982 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
4983 * @hba: per-adapter instance
4984 *
4985 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
4986 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
4987 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
4988 * This optimal value can help reduce the hibern8 exit latency.
4989 *
4990 * Returns zero on success, non-zero error value on failure.
4991 */
4992static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
4993{
4994        int ret = 0;
4995        u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
4996        u32 max_hibern8_time, tuned_pa_hibern8time;
4997
4998        ret = ufshcd_dme_get(hba,
4999                             UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
5000                                        UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
5001                                  &local_tx_hibern8_time_cap);
5002        if (ret)
5003                goto out;
5004
5005        ret = ufshcd_dme_peer_get(hba,
5006                                  UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
5007                                        UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
5008                                  &peer_rx_hibern8_time_cap);
5009        if (ret)
5010                goto out;
5011
5012        max_hibern8_time = max(local_tx_hibern8_time_cap,
5013                               peer_rx_hibern8_time_cap);
5014        /* make sure proper unit conversion is applied */
5015        tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
5016                                / PA_HIBERN8_TIME_UNIT_US);
5017        ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
5018                             tuned_pa_hibern8time);
5019out:
5020        return ret;
5021}
5022
5023/**
5024 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
5025 * less than device PA_TACTIVATE time.
5026 * @hba: per-adapter instance
5027 *
5028 * Some UFS devices require host PA_TACTIVATE to be lower than device
5029 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
5030 * for such devices.
5031 *
5032 * Returns zero on success, non-zero error value on failure.
5033 */
5034static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
5035{
5036        int ret = 0;
5037        u32 granularity, peer_granularity;
5038        u32 pa_tactivate, peer_pa_tactivate;
5039        u32 pa_tactivate_us, peer_pa_tactivate_us;
5040        u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
5041
5042        ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
5043                                  &granularity);
5044        if (ret)
5045                goto out;
5046
5047        ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
5048                                  &peer_granularity);
5049        if (ret)
5050                goto out;
5051
5052        if ((granularity < PA_GRANULARITY_MIN_VAL) ||
5053            (granularity > PA_GRANULARITY_MAX_VAL)) {
5054                dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
5055                        __func__, granularity);
5056                return -EINVAL;
5057        }
5058
5059        if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
5060            (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
5061                dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
5062                        __func__, peer_granularity);
5063                return -EINVAL;
5064        }
5065
5066        ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
5067        if (ret)
5068                goto out;
5069
5070        ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
5071                                  &peer_pa_tactivate);
5072        if (ret)
5073                goto out;
5074
5075        pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
5076        peer_pa_tactivate_us = peer_pa_tactivate *
5077                             gran_to_us_table[peer_granularity - 1];
5078
5079        if (pa_tactivate_us > peer_pa_tactivate_us) {
5080                u32 new_peer_pa_tactivate;
5081
5082                new_peer_pa_tactivate = pa_tactivate_us /
5083                                      gran_to_us_table[peer_granularity - 1];
5084                new_peer_pa_tactivate++;
5085                ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
5086                                          new_peer_pa_tactivate);
5087        }
5088
5089out:
5090        return ret;
5091}
5092
5093static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
5094{
5095        if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
5096                ufshcd_tune_pa_tactivate(hba);
5097                ufshcd_tune_pa_hibern8time(hba);
5098        }
5099
5100        if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
5101                /* set 1ms timeout for PA_TACTIVATE */
5102                ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
5103
5104        if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
5105                ufshcd_quirk_tune_host_pa_tactivate(hba);
5106
5107        ufshcd_vops_apply_dev_quirks(hba);
5108}
5109
5110/**
5111 * ufshcd_probe_hba - probe hba to detect device and initialize
5112 * @hba: per-adapter instance
5113 *
5114 * Execute link-startup and verify device initialization
5115 */
5116static int ufshcd_probe_hba(struct ufs_hba *hba)
5117{
5118        int ret;
5119
5120        ret = ufshcd_link_startup(hba);
5121        if (ret)
5122                goto out;
5123
5124        ufshcd_init_pwr_info(hba);
5125
5126        /* set the default level for urgent bkops */
5127        hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5128        hba->is_urgent_bkops_lvl_checked = false;
5129
5130        /* UniPro link is active now */
5131        ufshcd_set_link_active(hba);
5132
5133        ret = ufshcd_verify_dev_init(hba);
5134        if (ret)
5135                goto out;
5136
5137        ret = ufshcd_complete_dev_init(hba);
5138        if (ret)
5139                goto out;
5140
5141        ufs_advertise_fixup_device(hba);
5142        ufshcd_tune_unipro_params(hba);
5143
5144        ret = ufshcd_set_vccq_rail_unused(hba,
5145                (hba->dev_quirks & UFS_DEVICE_NO_VCCQ) ? true : false);
5146        if (ret)
5147                goto out;
5148
5149        /* UFS device is also active now */
5150        ufshcd_set_ufs_dev_active(hba);
5151        ufshcd_force_reset_auto_bkops(hba);
5152        hba->wlun_dev_clr_ua = true;
5153
5154        if (ufshcd_get_max_pwr_mode(hba)) {
5155                dev_err(hba->dev,
5156                        "%s: Failed getting max supported power mode\n",
5157                        __func__);
5158        } else {
5159                ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
5160                if (ret)
5161                        dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
5162                                        __func__, ret);
5163        }
5164
5165        /* set the state as operational after switching to desired gear */
5166        hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5167        /*
5168         * If we are in error handling context or in power management callbacks
5169         * context, no need to scan the host
5170         */
5171        if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
5172                bool flag;
5173
5174                /* clear any previous UFS device information */
5175                memset(&hba->dev_info, 0, sizeof(hba->dev_info));
5176                if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
5177                                QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
5178                        hba->dev_info.f_power_on_wp_en = flag;
5179
5180                if (!hba->is_init_prefetch)
5181                        ufshcd_init_icc_levels(hba);
5182
5183                /* Add required well known logical units to scsi mid layer */
5184                if (ufshcd_scsi_add_wlus(hba))
5185                        goto out;
5186
5187                scsi_scan_host(hba->host);
5188                pm_runtime_put_sync(hba->dev);
5189        }
5190
5191        if (!hba->is_init_prefetch)
5192                hba->is_init_prefetch = true;
5193
5194        /* Resume devfreq after UFS device is detected */
5195        if (ufshcd_is_clkscaling_enabled(hba))
5196                devfreq_resume_device(hba->devfreq);
5197
5198out:
5199        /*
5200         * If we failed to initialize the device or the device is not
5201         * present, turn off the power/clocks etc.
5202         */
5203        if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
5204                pm_runtime_put_sync(hba->dev);
5205                ufshcd_hba_exit(hba);
5206        }
5207
5208        return ret;
5209}
5210
5211/**
5212 * ufshcd_async_scan - asynchronous execution for probing hba
5213 * @data: data pointer to pass to this function
5214 * @cookie: cookie data
5215 */
5216static void ufshcd_async_scan(void *data, async_cookie_t cookie)
5217{
5218        struct ufs_hba *hba = (struct ufs_hba *)data;
5219
5220        ufshcd_probe_hba(hba);
5221}
5222
5223static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
5224{
5225        unsigned long flags;
5226        struct Scsi_Host *host;
5227        struct ufs_hba *hba;
5228        int index;
5229        bool found = false;
5230
5231        if (!scmd || !scmd->device || !scmd->device->host)
5232                return BLK_EH_NOT_HANDLED;
5233
5234        host = scmd->device->host;
5235        hba = shost_priv(host);
5236        if (!hba)
5237                return BLK_EH_NOT_HANDLED;
5238
5239        spin_lock_irqsave(host->host_lock, flags);
5240
5241        for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
5242                if (hba->lrb[index].cmd == scmd) {
5243                        found = true;
5244                        break;
5245                }
5246        }
5247
5248        spin_unlock_irqrestore(host->host_lock, flags);
5249
5250        /*
5251         * Bypass SCSI error handling and reset the block layer timer if this
5252         * SCSI command was not actually dispatched to UFS driver, otherwise
5253         * let SCSI layer handle the error as usual.
5254         */
5255        return found ? BLK_EH_NOT_HANDLED : BLK_EH_RESET_TIMER;
5256}
5257
5258static struct scsi_host_template ufshcd_driver_template = {
5259        .module                 = THIS_MODULE,
5260        .name                   = UFSHCD,
5261        .proc_name              = UFSHCD,
5262        .queuecommand           = ufshcd_queuecommand,
5263        .slave_alloc            = ufshcd_slave_alloc,
5264        .slave_configure        = ufshcd_slave_configure,
5265        .slave_destroy          = ufshcd_slave_destroy,
5266        .change_queue_depth     = ufshcd_change_queue_depth,
5267        .eh_abort_handler       = ufshcd_abort,
5268        .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
5269        .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
5270        .eh_timed_out           = ufshcd_eh_timed_out,
5271        .this_id                = -1,
5272        .sg_tablesize           = SG_ALL,
5273        .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
5274        .can_queue              = UFSHCD_CAN_QUEUE,
5275        .max_host_blocked       = 1,
5276        .track_queue_depth      = 1,
5277};
5278
5279static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
5280                                   int ua)
5281{
5282        int ret;
5283
5284        if (!vreg)
5285                return 0;
5286
5287        ret = regulator_set_load(vreg->reg, ua);
5288        if (ret < 0) {
5289                dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
5290                                __func__, vreg->name, ua, ret);
5291        }
5292
5293        return ret;
5294}
5295
5296static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
5297                                         struct ufs_vreg *vreg)
5298{
5299        if (!vreg)
5300                return 0;
5301        else if (vreg->unused)
5302                return 0;
5303        else
5304                return ufshcd_config_vreg_load(hba->dev, vreg,
5305                                               UFS_VREG_LPM_LOAD_UA);
5306}
5307
5308static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
5309                                         struct ufs_vreg *vreg)
5310{
5311        if (!vreg)
5312                return 0;
5313        else if (vreg->unused)
5314                return 0;
5315        else
5316                return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
5317}
5318
5319static int ufshcd_config_vreg(struct device *dev,
5320                struct ufs_vreg *vreg, bool on)
5321{
5322        int ret = 0;
5323        struct regulator *reg = vreg->reg;
5324        const char *name = vreg->name;
5325        int min_uV, uA_load;
5326
5327        BUG_ON(!vreg);
5328
5329        if (regulator_count_voltages(reg) > 0) {
5330                min_uV = on ? vreg->min_uV : 0;
5331                ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
5332                if (ret) {
5333                        dev_err(dev, "%s: %s set voltage failed, err=%d\n",
5334                                        __func__, name, ret);
5335                        goto out;
5336                }
5337
5338                uA_load = on ? vreg->max_uA : 0;
5339                ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
5340                if (ret)
5341                        goto out;
5342        }
5343out:
5344        return ret;
5345}
5346
5347static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
5348{
5349        int ret = 0;
5350
5351        if (!vreg)
5352                goto out;
5353        else if (vreg->enabled || vreg->unused)
5354                goto out;
5355
5356        ret = ufshcd_config_vreg(dev, vreg, true);
5357        if (!ret)
5358                ret = regulator_enable(vreg->reg);
5359
5360        if (!ret)
5361                vreg->enabled = true;
5362        else
5363                dev_err(dev, "%s: %s enable failed, err=%d\n",
5364                                __func__, vreg->name, ret);
5365out:
5366        return ret;
5367}
5368
5369static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
5370{
5371        int ret = 0;
5372
5373        if (!vreg)
5374                goto out;
5375        else if (!vreg->enabled || vreg->unused)
5376                goto out;
5377
5378        ret = regulator_disable(vreg->reg);
5379
5380        if (!ret) {
5381                /* ignore errors on applying disable config */
5382                ufshcd_config_vreg(dev, vreg, false);
5383                vreg->enabled = false;
5384        } else {
5385                dev_err(dev, "%s: %s disable failed, err=%d\n",
5386                                __func__, vreg->name, ret);
5387        }
5388out:
5389        return ret;
5390}
5391
5392static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
5393{
5394        int ret = 0;
5395        struct device *dev = hba->dev;
5396        struct ufs_vreg_info *info = &hba->vreg_info;
5397
5398        if (!info)
5399                goto out;
5400
5401        ret = ufshcd_toggle_vreg(dev, info->vcc, on);
5402        if (ret)
5403                goto out;
5404
5405        ret = ufshcd_toggle_vreg(dev, info->vccq, on);
5406        if (ret)
5407                goto out;
5408
5409        ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
5410        if (ret)
5411                goto out;
5412
5413out:
5414        if (ret) {
5415                ufshcd_toggle_vreg(dev, info->vccq2, false);
5416                ufshcd_toggle_vreg(dev, info->vccq, false);
5417                ufshcd_toggle_vreg(dev, info->vcc, false);
5418        }
5419        return ret;
5420}
5421
5422static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
5423{
5424        struct ufs_vreg_info *info = &hba->vreg_info;
5425
5426        if (info)
5427                return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
5428
5429        return 0;
5430}
5431
5432static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
5433{
5434        int ret = 0;
5435
5436        if (!vreg)
5437                goto out;
5438
5439        vreg->reg = devm_regulator_get(dev, vreg->name);
5440        if (IS_ERR(vreg->reg)) {
5441                ret = PTR_ERR(vreg->reg);
5442                dev_err(dev, "%s: %s get failed, err=%d\n",
5443                                __func__, vreg->name, ret);
5444        }
5445out:
5446        return ret;
5447}
5448
5449static int ufshcd_init_vreg(struct ufs_hba *hba)
5450{
5451        int ret = 0;
5452        struct device *dev = hba->dev;
5453        struct ufs_vreg_info *info = &hba->vreg_info;
5454
5455        if (!info)
5456                goto out;
5457
5458        ret = ufshcd_get_vreg(dev, info->vcc);
5459        if (ret)
5460                goto out;
5461
5462        ret = ufshcd_get_vreg(dev, info->vccq);
5463        if (ret)
5464                goto out;
5465
5466        ret = ufshcd_get_vreg(dev, info->vccq2);
5467out:
5468        return ret;
5469}
5470
5471static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
5472{
5473        struct ufs_vreg_info *info = &hba->vreg_info;
5474
5475        if (info)
5476                return ufshcd_get_vreg(hba->dev, info->vdd_hba);
5477
5478        return 0;
5479}
5480
5481static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused)
5482{
5483        int ret = 0;
5484        struct ufs_vreg_info *info = &hba->vreg_info;
5485
5486        if (!info)
5487                goto out;
5488        else if (!info->vccq)
5489                goto out;
5490
5491        if (unused) {
5492                /* shut off the rail here */
5493                ret = ufshcd_toggle_vreg(hba->dev, info->vccq, false);
5494                /*
5495                 * Mark this rail as no longer used, so it doesn't get enabled
5496                 * later by mistake
5497                 */
5498                if (!ret)
5499                        info->vccq->unused = true;
5500        } else {
5501                /*
5502                 * rail should have been already enabled hence just make sure
5503                 * that unused flag is cleared.
5504                 */
5505                info->vccq->unused = false;
5506        }
5507out:
5508        return ret;
5509}
5510
5511static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
5512                                        bool skip_ref_clk)
5513{
5514        int ret = 0;
5515        struct ufs_clk_info *clki;
5516        struct list_head *head = &hba->clk_list_head;
5517        unsigned long flags;
5518
5519        if (!head || list_empty(head))
5520                goto out;
5521
5522        list_for_each_entry(clki, head, list) {
5523                if (!IS_ERR_OR_NULL(clki->clk)) {
5524                        if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
5525                                continue;
5526
5527                        if (on && !clki->enabled) {
5528                                ret = clk_prepare_enable(clki->clk);
5529                                if (ret) {
5530                                        dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
5531                                                __func__, clki->name, ret);
5532                                        goto out;
5533                                }
5534                        } else if (!on && clki->enabled) {
5535                                clk_disable_unprepare(clki->clk);
5536                        }
5537                        clki->enabled = on;
5538                        dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
5539                                        clki->name, on ? "en" : "dis");
5540                }
5541        }
5542
5543        ret = ufshcd_vops_setup_clocks(hba, on);
5544out:
5545        if (ret) {
5546                list_for_each_entry(clki, head, list) {
5547                        if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
5548                                clk_disable_unprepare(clki->clk);
5549                }
5550        } else if (on) {
5551                spin_lock_irqsave(hba->host->host_lock, flags);
5552                hba->clk_gating.state = CLKS_ON;
5553                spin_unlock_irqrestore(hba->host->host_lock, flags);
5554        }
5555        return ret;
5556}
5557
5558static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
5559{
5560        return  __ufshcd_setup_clocks(hba, on, false);
5561}
5562
5563static int ufshcd_init_clocks(struct ufs_hba *hba)
5564{
5565        int ret = 0;
5566        struct ufs_clk_info *clki;
5567        struct device *dev = hba->dev;
5568        struct list_head *head = &hba->clk_list_head;
5569
5570        if (!head || list_empty(head))
5571                goto out;
5572
5573        list_for_each_entry(clki, head, list) {
5574                if (!clki->name)
5575                        continue;
5576
5577                clki->clk = devm_clk_get(dev, clki->name);
5578                if (IS_ERR(clki->clk)) {
5579                        ret = PTR_ERR(clki->clk);
5580                        dev_err(dev, "%s: %s clk get failed, %d\n",
5581                                        __func__, clki->name, ret);
5582                        goto out;
5583                }
5584
5585                if (clki->max_freq) {
5586                        ret = clk_set_rate(clki->clk, clki->max_freq);
5587                        if (ret) {
5588                                dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
5589                                        __func__, clki->name,
5590                                        clki->max_freq, ret);
5591                                goto out;
5592                        }
5593                        clki->curr_freq = clki->max_freq;
5594                }
5595                dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
5596                                clki->name, clk_get_rate(clki->clk));
5597        }
5598out:
5599        return ret;
5600}
5601
5602static int ufshcd_variant_hba_init(struct ufs_hba *hba)
5603{
5604        int err = 0;
5605
5606        if (!hba->vops)
5607                goto out;
5608
5609        err = ufshcd_vops_init(hba);
5610        if (err)
5611                goto out;
5612
5613        err = ufshcd_vops_setup_regulators(hba, true);
5614        if (err)
5615                goto out_exit;
5616
5617        goto out;
5618
5619out_exit:
5620        ufshcd_vops_exit(hba);
5621out:
5622        if (err)
5623                dev_err(hba->dev, "%s: variant %s init failed err %d\n",
5624                        __func__, ufshcd_get_var_name(hba), err);
5625        return err;
5626}
5627
5628static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
5629{
5630        if (!hba->vops)
5631                return;
5632
5633        ufshcd_vops_setup_clocks(hba, false);
5634
5635        ufshcd_vops_setup_regulators(hba, false);
5636
5637        ufshcd_vops_exit(hba);
5638}
5639
5640static int ufshcd_hba_init(struct ufs_hba *hba)
5641{
5642        int err;
5643
5644        /*
5645         * Handle host controller power separately from the UFS device power
5646         * rails as it will help controlling the UFS host controller power
5647         * collapse easily which is different than UFS device power collapse.
5648         * Also, enable the host controller power before we go ahead with rest
5649         * of the initialization here.
5650         */
5651        err = ufshcd_init_hba_vreg(hba);
5652        if (err)
5653                goto out;
5654
5655        err = ufshcd_setup_hba_vreg(hba, true);
5656        if (err)
5657                goto out;
5658
5659        err = ufshcd_init_clocks(hba);
5660        if (err)
5661                goto out_disable_hba_vreg;
5662
5663        err = ufshcd_setup_clocks(hba, true);
5664        if (err)
5665                goto out_disable_hba_vreg;
5666
5667        err = ufshcd_init_vreg(hba);
5668        if (err)
5669                goto out_disable_clks;
5670
5671        err = ufshcd_setup_vreg(hba, true);
5672        if (err)
5673                goto out_disable_clks;
5674
5675        err = ufshcd_variant_hba_init(hba);
5676        if (err)
5677                goto out_disable_vreg;
5678
5679        hba->is_powered = true;
5680        goto out;
5681
5682out_disable_vreg:
5683        ufshcd_setup_vreg(hba, false);
5684out_disable_clks:
5685        ufshcd_setup_clocks(hba, false);
5686out_disable_hba_vreg:
5687        ufshcd_setup_hba_vreg(hba, false);
5688out:
5689        return err;
5690}
5691
5692static void ufshcd_hba_exit(struct ufs_hba *hba)
5693{
5694        if (hba->is_powered) {
5695                ufshcd_variant_hba_exit(hba);
5696                ufshcd_setup_vreg(hba, false);
5697                ufshcd_setup_clocks(hba, false);
5698                ufshcd_setup_hba_vreg(hba, false);
5699                hba->is_powered = false;
5700        }
5701}
5702
5703static int
5704ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
5705{
5706        unsigned char cmd[6] = {REQUEST_SENSE,
5707                                0,
5708                                0,
5709                                0,
5710                                SCSI_SENSE_BUFFERSIZE,
5711                                0};
5712        char *buffer;
5713        int ret;
5714
5715        buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
5716        if (!buffer) {
5717                ret = -ENOMEM;
5718                goto out;
5719        }
5720
5721        ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer,
5722                                SCSI_SENSE_BUFFERSIZE, NULL,
5723                                msecs_to_jiffies(1000), 3, NULL, REQ_PM);
5724        if (ret)
5725                pr_err("%s: failed with err %d\n", __func__, ret);
5726
5727        kfree(buffer);
5728out:
5729        return ret;
5730}
5731
5732/**
5733 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
5734 *                           power mode
5735 * @hba: per adapter instance
5736 * @pwr_mode: device power mode to set
5737 *
5738 * Returns 0 if requested power mode is set successfully
5739 * Returns non-zero if failed to set the requested power mode
5740 */
5741static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
5742                                     enum ufs_dev_pwr_mode pwr_mode)
5743{
5744        unsigned char cmd[6] = { START_STOP };
5745        struct scsi_sense_hdr sshdr;
5746        struct scsi_device *sdp;
5747        unsigned long flags;
5748        int ret;
5749
5750        spin_lock_irqsave(hba->host->host_lock, flags);
5751        sdp = hba->sdev_ufs_device;
5752        if (sdp) {
5753                ret = scsi_device_get(sdp);
5754                if (!ret && !scsi_device_online(sdp)) {
5755                        ret = -ENODEV;
5756                        scsi_device_put(sdp);
5757                }
5758        } else {
5759                ret = -ENODEV;
5760        }
5761        spin_unlock_irqrestore(hba->host->host_lock, flags);
5762
5763        if (ret)
5764                return ret;
5765
5766        /*
5767         * If scsi commands fail, the scsi mid-layer schedules scsi error-
5768         * handling, which would wait for host to be resumed. Since we know
5769         * we are functional while we are here, skip host resume in error
5770         * handling context.
5771         */
5772        hba->host->eh_noresume = 1;
5773        if (hba->wlun_dev_clr_ua) {
5774                ret = ufshcd_send_request_sense(hba, sdp);
5775                if (ret)
5776                        goto out;
5777                /* Unit attention condition is cleared now */
5778                hba->wlun_dev_clr_ua = false;
5779        }
5780
5781        cmd[4] = pwr_mode << 4;
5782
5783        /*
5784         * Current function would be generally called from the power management
5785         * callbacks hence set the REQ_PM flag so that it doesn't resume the
5786         * already suspended childs.
5787         */
5788        ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
5789                                     START_STOP_TIMEOUT, 0, NULL, REQ_PM);
5790        if (ret) {
5791                sdev_printk(KERN_WARNING, sdp,
5792                            "START_STOP failed for power mode: %d, result %x\n",
5793                            pwr_mode, ret);
5794                if (driver_byte(ret) & DRIVER_SENSE)
5795                        scsi_print_sense_hdr(sdp, NULL, &sshdr);
5796        }
5797
5798        if (!ret)
5799                hba->curr_dev_pwr_mode = pwr_mode;
5800out:
5801        scsi_device_put(sdp);
5802        hba->host->eh_noresume = 0;
5803        return ret;
5804}
5805
5806static int ufshcd_link_state_transition(struct ufs_hba *hba,
5807                                        enum uic_link_state req_link_state,
5808                                        int check_for_bkops)
5809{
5810        int ret = 0;
5811
5812        if (req_link_state == hba->uic_link_state)
5813                return 0;
5814
5815        if (req_link_state == UIC_LINK_HIBERN8_STATE) {
5816                ret = ufshcd_uic_hibern8_enter(hba);
5817                if (!ret)
5818                        ufshcd_set_link_hibern8(hba);
5819                else
5820                        goto out;
5821        }
5822        /*
5823         * If autobkops is enabled, link can't be turned off because
5824         * turning off the link would also turn off the device.
5825         */
5826        else if ((req_link_state == UIC_LINK_OFF_STATE) &&
5827                   (!check_for_bkops || (check_for_bkops &&
5828                    !hba->auto_bkops_enabled))) {
5829                /*
5830                 * Let's make sure that link is in low power mode, we are doing
5831                 * this currently by putting the link in Hibern8. Otherway to
5832                 * put the link in low power mode is to send the DME end point
5833                 * to device and then send the DME reset command to local
5834                 * unipro. But putting the link in hibern8 is much faster.
5835                 */
5836                ret = ufshcd_uic_hibern8_enter(hba);
5837                if (ret)
5838                        goto out;
5839                /*
5840                 * Change controller state to "reset state" which
5841                 * should also put the link in off/reset state
5842                 */
5843                ufshcd_hba_stop(hba, true);
5844                /*
5845                 * TODO: Check if we need any delay to make sure that
5846                 * controller is reset
5847                 */
5848                ufshcd_set_link_off(hba);
5849        }
5850
5851out:
5852        return ret;
5853}
5854
5855static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
5856{
5857        /*
5858         * It seems some UFS devices may keep drawing more than sleep current
5859         * (atleast for 500us) from UFS rails (especially from VCCQ rail).
5860         * To avoid this situation, add 2ms delay before putting these UFS
5861         * rails in LPM mode.
5862         */
5863        if (!ufshcd_is_link_active(hba) &&
5864            hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
5865                usleep_range(2000, 2100);
5866
5867        /*
5868         * If UFS device is either in UFS_Sleep turn off VCC rail to save some
5869         * power.
5870         *
5871         * If UFS device and link is in OFF state, all power supplies (VCC,
5872         * VCCQ, VCCQ2) can be turned off if power on write protect is not
5873         * required. If UFS link is inactive (Hibern8 or OFF state) and device
5874         * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
5875         *
5876         * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
5877         * in low power state which would save some power.
5878         */
5879        if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5880            !hba->dev_info.is_lu_power_on_wp) {
5881                ufshcd_setup_vreg(hba, false);
5882        } else if (!ufshcd_is_ufs_dev_active(hba)) {
5883                ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5884                if (!ufshcd_is_link_active(hba)) {
5885                        ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5886                        ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
5887                }
5888        }
5889}
5890
5891static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
5892{
5893        int ret = 0;
5894
5895        if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5896            !hba->dev_info.is_lu_power_on_wp) {
5897                ret = ufshcd_setup_vreg(hba, true);
5898        } else if (!ufshcd_is_ufs_dev_active(hba)) {
5899                ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
5900                if (!ret && !ufshcd_is_link_active(hba)) {
5901                        ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5902                        if (ret)
5903                                goto vcc_disable;
5904                        ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5905                        if (ret)
5906                                goto vccq_lpm;
5907                }
5908        }
5909        goto out;
5910
5911vccq_lpm:
5912        ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5913vcc_disable:
5914        ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5915out:
5916        return ret;
5917}
5918
5919static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
5920{
5921        if (ufshcd_is_link_off(hba))
5922                ufshcd_setup_hba_vreg(hba, false);
5923}
5924
5925static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
5926{
5927        if (ufshcd_is_link_off(hba))
5928                ufshcd_setup_hba_vreg(hba, true);
5929}
5930
5931/**
5932 * ufshcd_suspend - helper function for suspend operations
5933 * @hba: per adapter instance
5934 * @pm_op: desired low power operation type
5935 *
5936 * This function will try to put the UFS device and link into low power
5937 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
5938 * (System PM level).
5939 *
5940 * If this function is called during shutdown, it will make sure that
5941 * both UFS device and UFS link is powered off.
5942 *
5943 * NOTE: UFS device & link must be active before we enter in this function.
5944 *
5945 * Returns 0 for success and non-zero for failure
5946 */
5947static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
5948{
5949        int ret = 0;
5950        enum ufs_pm_level pm_lvl;
5951        enum ufs_dev_pwr_mode req_dev_pwr_mode;
5952        enum uic_link_state req_link_state;
5953
5954        hba->pm_op_in_progress = 1;
5955        if (!ufshcd_is_shutdown_pm(pm_op)) {
5956                pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
5957                         hba->rpm_lvl : hba->spm_lvl;
5958                req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
5959                req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
5960        } else {
5961                req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
5962                req_link_state = UIC_LINK_OFF_STATE;
5963        }
5964
5965        /*
5966         * If we can't transition into any of the low power modes
5967         * just gate the clocks.
5968         */
5969        ufshcd_hold(hba, false);
5970        hba->clk_gating.is_suspended = true;
5971
5972        if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
5973                        req_link_state == UIC_LINK_ACTIVE_STATE) {
5974                goto disable_clks;
5975        }
5976
5977        if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
5978            (req_link_state == hba->uic_link_state))
5979                goto out;
5980
5981        /* UFS device & link must be active before we enter in this function */
5982        if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
5983                ret = -EINVAL;
5984                goto out;
5985        }
5986
5987        if (ufshcd_is_runtime_pm(pm_op)) {
5988                if (ufshcd_can_autobkops_during_suspend(hba)) {
5989                        /*
5990                         * The device is idle with no requests in the queue,
5991                         * allow background operations if bkops status shows
5992                         * that performance might be impacted.
5993                         */
5994                        ret = ufshcd_urgent_bkops(hba);
5995                        if (ret)
5996                                goto enable_gating;
5997                } else {
5998                        /* make sure that auto bkops is disabled */
5999                        ufshcd_disable_auto_bkops(hba);
6000                }
6001        }
6002
6003        if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
6004             ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
6005               !ufshcd_is_runtime_pm(pm_op))) {
6006                /* ensure that bkops is disabled */
6007                ufshcd_disable_auto_bkops(hba);
6008                ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
6009                if (ret)
6010                        goto enable_gating;
6011        }
6012
6013        ret = ufshcd_link_state_transition(hba, req_link_state, 1);
6014        if (ret)
6015                goto set_dev_active;
6016
6017        ufshcd_vreg_set_lpm(hba);
6018
6019disable_clks:
6020        /*
6021         * The clock scaling needs access to controller registers. Hence, Wait
6022         * for pending clock scaling work to be done before clocks are
6023         * turned off.
6024         */
6025        if (ufshcd_is_clkscaling_enabled(hba)) {
6026                devfreq_suspend_device(hba->devfreq);
6027                hba->clk_scaling.window_start_t = 0;
6028        }
6029        /*
6030         * Call vendor specific suspend callback. As these callbacks may access
6031         * vendor specific host controller register space call them before the
6032         * host clocks are ON.
6033         */
6034        ret = ufshcd_vops_suspend(hba, pm_op);
6035        if (ret)
6036                goto set_link_active;
6037
6038        ret = ufshcd_vops_setup_clocks(hba, false);
6039        if (ret)
6040                goto vops_resume;
6041
6042        if (!ufshcd_is_link_active(hba))
6043                ufshcd_setup_clocks(hba, false);
6044        else
6045                /* If link is active, device ref_clk can't be switched off */
6046                __ufshcd_setup_clocks(hba, false, true);
6047
6048        hba->clk_gating.state = CLKS_OFF;
6049        /*
6050         * Disable the host irq as host controller as there won't be any
6051         * host controller transaction expected till resume.
6052         */
6053        ufshcd_disable_irq(hba);
6054        /* Put the host controller in low power mode if possible */
6055        ufshcd_hba_vreg_set_lpm(hba);
6056        goto out;
6057
6058vops_resume:
6059        ufshcd_vops_resume(hba, pm_op);
6060set_link_active:
6061        ufshcd_vreg_set_hpm(hba);
6062        if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
6063                ufshcd_set_link_active(hba);
6064        else if (ufshcd_is_link_off(hba))
6065                ufshcd_host_reset_and_restore(hba);
6066set_dev_active:
6067        if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
6068                ufshcd_disable_auto_bkops(hba);
6069enable_gating:
6070        hba->clk_gating.is_suspended = false;
6071        ufshcd_release(hba);
6072out:
6073        hba->pm_op_in_progress = 0;
6074        return ret;
6075}
6076
6077/**
6078 * ufshcd_resume - helper function for resume operations
6079 * @hba: per adapter instance
6080 * @pm_op: runtime PM or system PM
6081 *
6082 * This function basically brings the UFS device, UniPro link and controller
6083 * to active state.
6084 *
6085 * Returns 0 for success and non-zero for failure
6086 */
6087static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
6088{
6089        int ret;
6090        enum uic_link_state old_link_state;
6091
6092        hba->pm_op_in_progress = 1;
6093        old_link_state = hba->uic_link_state;
6094
6095        ufshcd_hba_vreg_set_hpm(hba);
6096        /* Make sure clocks are enabled before accessing controller */
6097        ret = ufshcd_setup_clocks(hba, true);
6098        if (ret)
6099                goto out;
6100
6101        /* enable the host irq as host controller would be active soon */
6102        ret = ufshcd_enable_irq(hba);
6103        if (ret)
6104                goto disable_irq_and_vops_clks;
6105
6106        ret = ufshcd_vreg_set_hpm(hba);
6107        if (ret)
6108                goto disable_irq_and_vops_clks;
6109
6110        /*
6111         * Call vendor specific resume callback. As these callbacks may access
6112         * vendor specific host controller register space call them when the
6113         * host clocks are ON.
6114         */
6115        ret = ufshcd_vops_resume(hba, pm_op);
6116        if (ret)
6117                goto disable_vreg;
6118
6119        if (ufshcd_is_link_hibern8(hba)) {
6120                ret = ufshcd_uic_hibern8_exit(hba);
6121                if (!ret)
6122                        ufshcd_set_link_active(hba);
6123                else
6124                        goto vendor_suspend;
6125        } else if (ufshcd_is_link_off(hba)) {
6126                ret = ufshcd_host_reset_and_restore(hba);
6127                /*
6128                 * ufshcd_host_reset_and_restore() should have already
6129                 * set the link state as active
6130                 */
6131                if (ret || !ufshcd_is_link_active(hba))
6132                        goto vendor_suspend;
6133        }
6134
6135        if (!ufshcd_is_ufs_dev_active(hba)) {
6136                ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
6137                if (ret)
6138                        goto set_old_link_state;
6139        }
6140
6141        /*
6142         * If BKOPs operations are urgently needed at this moment then
6143         * keep auto-bkops enabled or else disable it.
6144         */
6145        ufshcd_urgent_bkops(hba);
6146        hba->clk_gating.is_suspended = false;
6147
6148        if (ufshcd_is_clkscaling_enabled(hba))
6149                devfreq_resume_device(hba->devfreq);
6150
6151        /* Schedule clock gating in case of no access to UFS device yet */
6152        ufshcd_release(hba);
6153        goto out;
6154
6155set_old_link_state:
6156        ufshcd_link_state_transition(hba, old_link_state, 0);
6157vendor_suspend:
6158        ufshcd_vops_suspend(hba, pm_op);
6159disable_vreg:
6160        ufshcd_vreg_set_lpm(hba);
6161disable_irq_and_vops_clks:
6162        ufshcd_disable_irq(hba);
6163        ufshcd_setup_clocks(hba, false);
6164out:
6165        hba->pm_op_in_progress = 0;
6166        return ret;
6167}
6168
6169/**
6170 * ufshcd_system_suspend - system suspend routine
6171 * @hba: per adapter instance
6172 * @pm_op: runtime PM or system PM
6173 *
6174 * Check the description of ufshcd_suspend() function for more details.
6175 *
6176 * Returns 0 for success and non-zero for failure
6177 */
6178int ufshcd_system_suspend(struct ufs_hba *hba)
6179{
6180        int ret = 0;
6181
6182        if (!hba || !hba->is_powered)
6183                return 0;
6184
6185        if (pm_runtime_suspended(hba->dev)) {
6186                if (hba->rpm_lvl == hba->spm_lvl)
6187                        /*
6188                         * There is possibility that device may still be in
6189                         * active state during the runtime suspend.
6190                         */
6191                        if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
6192                            hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled)
6193                                goto out;
6194
6195                /*
6196                 * UFS device and/or UFS link low power states during runtime
6197                 * suspend seems to be different than what is expected during
6198                 * system suspend. Hence runtime resume the devic & link and
6199                 * let the system suspend low power states to take effect.
6200                 * TODO: If resume takes longer time, we might have optimize
6201                 * it in future by not resuming everything if possible.
6202                 */
6203                ret = ufshcd_runtime_resume(hba);
6204                if (ret)
6205                        goto out;
6206        }
6207
6208        ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
6209out:
6210        if (!ret)
6211                hba->is_sys_suspended = true;
6212        return ret;
6213}
6214EXPORT_SYMBOL(ufshcd_system_suspend);
6215
6216/**
6217 * ufshcd_system_resume - system resume routine
6218 * @hba: per adapter instance
6219 *
6220 * Returns 0 for success and non-zero for failure
6221 */
6222
6223int ufshcd_system_resume(struct ufs_hba *hba)
6224{
6225        if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev))
6226                /*
6227                 * Let the runtime resume take care of resuming
6228                 * if runtime suspended.
6229                 */
6230                return 0;
6231
6232        return ufshcd_resume(hba, UFS_SYSTEM_PM);
6233}
6234EXPORT_SYMBOL(ufshcd_system_resume);
6235
6236/**
6237 * ufshcd_runtime_suspend - runtime suspend routine
6238 * @hba: per adapter instance
6239 *
6240 * Check the description of ufshcd_suspend() function for more details.
6241 *
6242 * Returns 0 for success and non-zero for failure
6243 */
6244int ufshcd_runtime_suspend(struct ufs_hba *hba)
6245{
6246        if (!hba || !hba->is_powered)
6247                return 0;
6248
6249        return ufshcd_suspend(hba, UFS_RUNTIME_PM);
6250}
6251EXPORT_SYMBOL(ufshcd_runtime_suspend);
6252
6253/**
6254 * ufshcd_runtime_resume - runtime resume routine
6255 * @hba: per adapter instance
6256 *
6257 * This function basically brings the UFS device, UniPro link and controller
6258 * to active state. Following operations are done in this function:
6259 *
6260 * 1. Turn on all the controller related clocks
6261 * 2. Bring the UniPro link out of Hibernate state
6262 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
6263 *    to active state.
6264 * 4. If auto-bkops is enabled on the device, disable it.
6265 *
6266 * So following would be the possible power state after this function return
6267 * successfully:
6268 *      S1: UFS device in Active state with VCC rail ON
6269 *          UniPro link in Active state
6270 *          All the UFS/UniPro controller clocks are ON
6271 *
6272 * Returns 0 for success and non-zero for failure
6273 */
6274int ufshcd_runtime_resume(struct ufs_hba *hba)
6275{
6276        if (!hba || !hba->is_powered)
6277                return 0;
6278        else
6279                return ufshcd_resume(hba, UFS_RUNTIME_PM);
6280}
6281EXPORT_SYMBOL(ufshcd_runtime_resume);
6282
6283int ufshcd_runtime_idle(struct ufs_hba *hba)
6284{
6285        return 0;
6286}
6287EXPORT_SYMBOL(ufshcd_runtime_idle);
6288
6289/**
6290 * ufshcd_shutdown - shutdown routine
6291 * @hba: per adapter instance
6292 *
6293 * This function would power off both UFS device and UFS link.
6294 *
6295 * Returns 0 always to allow force shutdown even in case of errors.
6296 */
6297int ufshcd_shutdown(struct ufs_hba *hba)
6298{
6299        int ret = 0;
6300
6301        if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
6302                goto out;
6303
6304        if (pm_runtime_suspended(hba->dev)) {
6305                ret = ufshcd_runtime_resume(hba);
6306                if (ret)
6307                        goto out;
6308        }
6309
6310        ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
6311out:
6312        if (ret)
6313                dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
6314        /* allow force shutdown even in case of errors */
6315        return 0;
6316}
6317EXPORT_SYMBOL(ufshcd_shutdown);
6318
6319/**
6320 * ufshcd_remove - de-allocate SCSI host and host memory space
6321 *              data structure memory
6322 * @hba - per adapter instance
6323 */
6324void ufshcd_remove(struct ufs_hba *hba)
6325{
6326        scsi_remove_host(hba->host);
6327        /* disable interrupts */
6328        ufshcd_disable_intr(hba, hba->intr_mask);
6329        ufshcd_hba_stop(hba, true);
6330
6331        scsi_host_put(hba->host);
6332
6333        ufshcd_exit_clk_gating(hba);
6334        if (ufshcd_is_clkscaling_enabled(hba))
6335                devfreq_remove_device(hba->devfreq);
6336        ufshcd_hba_exit(hba);
6337}
6338EXPORT_SYMBOL_GPL(ufshcd_remove);
6339
6340/**
6341 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
6342 * @hba: pointer to Host Bus Adapter (HBA)
6343 */
6344void ufshcd_dealloc_host(struct ufs_hba *hba)
6345{
6346        scsi_host_put(hba->host);
6347}
6348EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
6349
6350/**
6351 * ufshcd_set_dma_mask - Set dma mask based on the controller
6352 *                       addressing capability
6353 * @hba: per adapter instance
6354 *
6355 * Returns 0 for success, non-zero for failure
6356 */
6357static int ufshcd_set_dma_mask(struct ufs_hba *hba)
6358{
6359        if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
6360                if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
6361                        return 0;
6362        }
6363        return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
6364}
6365
6366/**
6367 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
6368 * @dev: pointer to device handle
6369 * @hba_handle: driver private handle
6370 * Returns 0 on success, non-zero value on failure
6371 */
6372int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
6373{
6374        struct Scsi_Host *host;
6375        struct ufs_hba *hba;
6376        int err = 0;
6377
6378        if (!dev) {
6379                dev_err(dev,
6380                "Invalid memory reference for dev is NULL\n");
6381                err = -ENODEV;
6382                goto out_error;
6383        }
6384
6385        host = scsi_host_alloc(&ufshcd_driver_template,
6386                                sizeof(struct ufs_hba));
6387        if (!host) {
6388                dev_err(dev, "scsi_host_alloc failed\n");
6389                err = -ENOMEM;
6390                goto out_error;
6391        }
6392        hba = shost_priv(host);
6393        hba->host = host;
6394        hba->dev = dev;
6395        *hba_handle = hba;
6396
6397out_error:
6398        return err;
6399}
6400EXPORT_SYMBOL(ufshcd_alloc_host);
6401
6402static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
6403{
6404        int ret = 0;
6405        struct ufs_clk_info *clki;
6406        struct list_head *head = &hba->clk_list_head;
6407
6408        if (!head || list_empty(head))
6409                goto out;
6410
6411        ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
6412        if (ret)
6413                return ret;
6414
6415        list_for_each_entry(clki, head, list) {
6416                if (!IS_ERR_OR_NULL(clki->clk)) {
6417                        if (scale_up && clki->max_freq) {
6418                                if (clki->curr_freq == clki->max_freq)
6419                                        continue;
6420                                ret = clk_set_rate(clki->clk, clki->max_freq);
6421                                if (ret) {
6422                                        dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
6423                                                __func__, clki->name,
6424                                                clki->max_freq, ret);
6425                                        break;
6426                                }
6427                                clki->curr_freq = clki->max_freq;
6428
6429                        } else if (!scale_up && clki->min_freq) {
6430                                if (clki->curr_freq == clki->min_freq)
6431                                        continue;
6432                                ret = clk_set_rate(clki->clk, clki->min_freq);
6433                                if (ret) {
6434                                        dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
6435                                                __func__, clki->name,
6436                                                clki->min_freq, ret);
6437                                        break;
6438                                }
6439                                clki->curr_freq = clki->min_freq;
6440                        }
6441                }
6442                dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
6443                                clki->name, clk_get_rate(clki->clk));
6444        }
6445
6446        ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
6447
6448out:
6449        return ret;
6450}
6451
6452static int ufshcd_devfreq_target(struct device *dev,
6453                                unsigned long *freq, u32 flags)
6454{
6455        int err = 0;
6456        struct ufs_hba *hba = dev_get_drvdata(dev);
6457
6458        if (!ufshcd_is_clkscaling_enabled(hba))
6459                return -EINVAL;
6460
6461        if (*freq == UINT_MAX)
6462                err = ufshcd_scale_clks(hba, true);
6463        else if (*freq == 0)
6464                err = ufshcd_scale_clks(hba, false);
6465
6466        return err;
6467}
6468
6469static int ufshcd_devfreq_get_dev_status(struct device *dev,
6470                struct devfreq_dev_status *stat)
6471{
6472        struct ufs_hba *hba = dev_get_drvdata(dev);
6473        struct ufs_clk_scaling *scaling = &hba->clk_scaling;
6474        unsigned long flags;
6475
6476        if (!ufshcd_is_clkscaling_enabled(hba))
6477                return -EINVAL;
6478
6479        memset(stat, 0, sizeof(*stat));
6480
6481        spin_lock_irqsave(hba->host->host_lock, flags);
6482        if (!scaling->window_start_t)
6483                goto start_window;
6484
6485        if (scaling->is_busy_started)
6486                scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
6487                                        scaling->busy_start_t));
6488
6489        stat->total_time = jiffies_to_usecs((long)jiffies -
6490                                (long)scaling->window_start_t);
6491        stat->busy_time = scaling->tot_busy_t;
6492start_window:
6493        scaling->window_start_t = jiffies;
6494        scaling->tot_busy_t = 0;
6495
6496        if (hba->outstanding_reqs) {
6497                scaling->busy_start_t = ktime_get();
6498                scaling->is_busy_started = true;
6499        } else {
6500                scaling->busy_start_t = ktime_set(0, 0);
6501                scaling->is_busy_started = false;
6502        }
6503        spin_unlock_irqrestore(hba->host->host_lock, flags);
6504        return 0;
6505}
6506
6507static struct devfreq_dev_profile ufs_devfreq_profile = {
6508        .polling_ms     = 100,
6509        .target         = ufshcd_devfreq_target,
6510        .get_dev_status = ufshcd_devfreq_get_dev_status,
6511};
6512
6513/**
6514 * ufshcd_init - Driver initialization routine
6515 * @hba: per-adapter instance
6516 * @mmio_base: base register address
6517 * @irq: Interrupt line of device
6518 * Returns 0 on success, non-zero value on failure
6519 */
6520int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
6521{
6522        int err;
6523        struct Scsi_Host *host = hba->host;
6524        struct device *dev = hba->dev;
6525
6526        if (!mmio_base) {
6527                dev_err(hba->dev,
6528                "Invalid memory reference for mmio_base is NULL\n");
6529                err = -ENODEV;
6530                goto out_error;
6531        }
6532
6533        hba->mmio_base = mmio_base;
6534        hba->irq = irq;
6535
6536        err = ufshcd_hba_init(hba);
6537        if (err)
6538                goto out_error;
6539
6540        /* Read capabilities registers */
6541        ufshcd_hba_capabilities(hba);
6542
6543        /* Get UFS version supported by the controller */
6544        hba->ufs_version = ufshcd_get_ufs_version(hba);
6545
6546        /* Get Interrupt bit mask per version */
6547        hba->intr_mask = ufshcd_get_intr_mask(hba);
6548
6549        err = ufshcd_set_dma_mask(hba);
6550        if (err) {
6551                dev_err(hba->dev, "set dma mask failed\n");
6552                goto out_disable;
6553        }
6554
6555        /* Allocate memory for host memory space */
6556        err = ufshcd_memory_alloc(hba);
6557        if (err) {
6558                dev_err(hba->dev, "Memory allocation failed\n");
6559                goto out_disable;
6560        }
6561
6562        /* Configure LRB */
6563        ufshcd_host_memory_configure(hba);
6564
6565        host->can_queue = hba->nutrs;
6566        host->cmd_per_lun = hba->nutrs;
6567        host->max_id = UFSHCD_MAX_ID;
6568        host->max_lun = UFS_MAX_LUNS;
6569        host->max_channel = UFSHCD_MAX_CHANNEL;
6570        host->unique_id = host->host_no;
6571        host->max_cmd_len = MAX_CDB_SIZE;
6572
6573        hba->max_pwr_info.is_valid = false;
6574
6575        /* Initailize wait queue for task management */
6576        init_waitqueue_head(&hba->tm_wq);
6577        init_waitqueue_head(&hba->tm_tag_wq);
6578
6579        /* Initialize work queues */
6580        INIT_WORK(&hba->eh_work, ufshcd_err_handler);
6581        INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
6582
6583        /* Initialize UIC command mutex */
6584        mutex_init(&hba->uic_cmd_mutex);
6585
6586        /* Initialize mutex for device management commands */
6587        mutex_init(&hba->dev_cmd.lock);
6588
6589        /* Initialize device management tag acquire wait queue */
6590        init_waitqueue_head(&hba->dev_cmd.tag_wq);
6591
6592        ufshcd_init_clk_gating(hba);
6593
6594        /*
6595         * In order to avoid any spurious interrupt immediately after
6596         * registering UFS controller interrupt handler, clear any pending UFS
6597         * interrupt status and disable all the UFS interrupts.
6598         */
6599        ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
6600                      REG_INTERRUPT_STATUS);
6601        ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
6602        /*
6603         * Make sure that UFS interrupts are disabled and any pending interrupt
6604         * status is cleared before registering UFS interrupt handler.
6605         */
6606        mb();
6607
6608        /* IRQ registration */
6609        err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
6610        if (err) {
6611                dev_err(hba->dev, "request irq failed\n");
6612                goto exit_gating;
6613        } else {
6614                hba->is_irq_enabled = true;
6615        }
6616
6617        err = scsi_add_host(host, hba->dev);
6618        if (err) {
6619                dev_err(hba->dev, "scsi_add_host failed\n");
6620                goto exit_gating;
6621        }
6622
6623        /* Host controller enable */
6624        err = ufshcd_hba_enable(hba);
6625        if (err) {
6626                dev_err(hba->dev, "Host controller enable failed\n");
6627                goto out_remove_scsi_host;
6628        }
6629
6630        if (ufshcd_is_clkscaling_enabled(hba)) {
6631                hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile,
6632                                                   "simple_ondemand", NULL);
6633                if (IS_ERR(hba->devfreq)) {
6634                        dev_err(hba->dev, "Unable to register with devfreq %ld\n",
6635                                        PTR_ERR(hba->devfreq));
6636                        err = PTR_ERR(hba->devfreq);
6637                        goto out_remove_scsi_host;
6638                }
6639                /* Suspend devfreq until the UFS device is detected */
6640                devfreq_suspend_device(hba->devfreq);
6641                hba->clk_scaling.window_start_t = 0;
6642        }
6643
6644        /* Hold auto suspend until async scan completes */
6645        pm_runtime_get_sync(dev);
6646
6647        /*
6648         * We are assuming that device wasn't put in sleep/power-down
6649         * state exclusively during the boot stage before kernel.
6650         * This assumption helps avoid doing link startup twice during
6651         * ufshcd_probe_hba().
6652         */
6653        ufshcd_set_ufs_dev_active(hba);
6654
6655        async_schedule(ufshcd_async_scan, hba);
6656
6657        return 0;
6658
6659out_remove_scsi_host:
6660        scsi_remove_host(hba->host);
6661exit_gating:
6662        ufshcd_exit_clk_gating(hba);
6663out_disable:
6664        hba->is_irq_enabled = false;
6665        scsi_host_put(host);
6666        ufshcd_hba_exit(hba);
6667out_error:
6668        return err;
6669}
6670EXPORT_SYMBOL_GPL(ufshcd_init);
6671
6672MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
6673MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
6674MODULE_DESCRIPTION("Generic UFS host controller driver Core");
6675MODULE_LICENSE("GPL");
6676MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Note: See TracBrowser for help on using the repository browser.