Merge branch 'fixes-v5.1-a' of git://git.kernel.org/pub/scm/linux/kernel/git/jmorris...
[muen/linux.git] / drivers / misc / habanalabs / device.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4  * Copyright 2016-2019 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7
8 #include "habanalabs.h"
9
10 #include <linux/pci.h>
11 #include <linux/sched/signal.h>
12 #include <linux/hwmon.h>
13
14 #define HL_PLDM_PENDING_RESET_PER_SEC   (HL_PENDING_RESET_PER_SEC * 10)
15
16 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
17 {
18         if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
19                 return true;
20         else
21                 return false;
22 }
23
24 static void hpriv_release(struct kref *ref)
25 {
26         struct hl_fpriv *hpriv;
27         struct hl_device *hdev;
28
29         hpriv = container_of(ref, struct hl_fpriv, refcount);
30
31         hdev = hpriv->hdev;
32
33         put_pid(hpriv->taskpid);
34
35         hl_debugfs_remove_file(hpriv);
36
37         mutex_destroy(&hpriv->restore_phase_mutex);
38
39         kfree(hpriv);
40
41         /* Now the FD is really closed */
42         atomic_dec(&hdev->fd_open_cnt);
43
44         /* This allows a new user context to open the device */
45         hdev->user_ctx = NULL;
46 }
47
48 void hl_hpriv_get(struct hl_fpriv *hpriv)
49 {
50         kref_get(&hpriv->refcount);
51 }
52
53 void hl_hpriv_put(struct hl_fpriv *hpriv)
54 {
55         kref_put(&hpriv->refcount, hpriv_release);
56 }
57
58 /*
59  * hl_device_release - release function for habanalabs device
60  *
61  * @inode: pointer to inode structure
62  * @filp: pointer to file structure
63  *
64  * Called when process closes an habanalabs device
65  */
66 static int hl_device_release(struct inode *inode, struct file *filp)
67 {
68         struct hl_fpriv *hpriv = filp->private_data;
69
70         hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
71         hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
72
73         filp->private_data = NULL;
74
75         hl_hpriv_put(hpriv);
76
77         return 0;
78 }
79
80 /*
81  * hl_mmap - mmap function for habanalabs device
82  *
83  * @*filp: pointer to file structure
84  * @*vma: pointer to vm_area_struct of the process
85  *
86  * Called when process does an mmap on habanalabs device. Call the device's mmap
87  * function at the end of the common code.
88  */
89 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
90 {
91         struct hl_fpriv *hpriv = filp->private_data;
92
93         if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
94                 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
95                 return hl_cb_mmap(hpriv, vma);
96         }
97
98         return -EINVAL;
99 }
100
101 static const struct file_operations hl_ops = {
102         .owner = THIS_MODULE,
103         .open = hl_device_open,
104         .release = hl_device_release,
105         .mmap = hl_mmap,
106         .unlocked_ioctl = hl_ioctl,
107         .compat_ioctl = hl_ioctl
108 };
109
110 /*
111  * device_setup_cdev - setup cdev and device for habanalabs device
112  *
113  * @hdev: pointer to habanalabs device structure
114  * @hclass: pointer to the class object of the device
115  * @minor: minor number of the specific device
116  * @fpos : file operations to install for this device
117  *
118  * Create a cdev and a Linux device for habanalabs's device. Need to be
119  * called at the end of the habanalabs device initialization process,
120  * because this function exposes the device to the user
121  */
122 static int device_setup_cdev(struct hl_device *hdev, struct class *hclass,
123                                 int minor, const struct file_operations *fops)
124 {
125         int err, devno = MKDEV(hdev->major, minor);
126         struct cdev *hdev_cdev = &hdev->cdev;
127         char *name;
128
129         name = kasprintf(GFP_KERNEL, "hl%d", hdev->id);
130         if (!name)
131                 return -ENOMEM;
132
133         cdev_init(hdev_cdev, fops);
134         hdev_cdev->owner = THIS_MODULE;
135         err = cdev_add(hdev_cdev, devno, 1);
136         if (err) {
137                 pr_err("Failed to add char device %s\n", name);
138                 goto err_cdev_add;
139         }
140
141         hdev->dev = device_create(hclass, NULL, devno, NULL, "%s", name);
142         if (IS_ERR(hdev->dev)) {
143                 pr_err("Failed to create device %s\n", name);
144                 err = PTR_ERR(hdev->dev);
145                 goto err_device_create;
146         }
147
148         dev_set_drvdata(hdev->dev, hdev);
149
150         kfree(name);
151
152         return 0;
153
154 err_device_create:
155         cdev_del(hdev_cdev);
156 err_cdev_add:
157         kfree(name);
158         return err;
159 }
160
161 /*
162  * device_early_init - do some early initialization for the habanalabs device
163  *
164  * @hdev: pointer to habanalabs device structure
165  *
166  * Install the relevant function pointers and call the early_init function,
167  * if such a function exists
168  */
169 static int device_early_init(struct hl_device *hdev)
170 {
171         int rc;
172
173         switch (hdev->asic_type) {
174         case ASIC_GOYA:
175                 goya_set_asic_funcs(hdev);
176                 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
177                 break;
178         default:
179                 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
180                         hdev->asic_type);
181                 return -EINVAL;
182         }
183
184         rc = hdev->asic_funcs->early_init(hdev);
185         if (rc)
186                 return rc;
187
188         rc = hl_asid_init(hdev);
189         if (rc)
190                 goto early_fini;
191
192         hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
193         if (hdev->cq_wq == NULL) {
194                 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
195                 rc = -ENOMEM;
196                 goto asid_fini;
197         }
198
199         hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
200         if (hdev->eq_wq == NULL) {
201                 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
202                 rc = -ENOMEM;
203                 goto free_cq_wq;
204         }
205
206         hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
207                                         GFP_KERNEL);
208         if (!hdev->hl_chip_info) {
209                 rc = -ENOMEM;
210                 goto free_eq_wq;
211         }
212
213         hl_cb_mgr_init(&hdev->kernel_cb_mgr);
214
215         mutex_init(&hdev->fd_open_cnt_lock);
216         mutex_init(&hdev->send_cpu_message_lock);
217         INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
218         spin_lock_init(&hdev->hw_queues_mirror_lock);
219         atomic_set(&hdev->in_reset, 0);
220         atomic_set(&hdev->fd_open_cnt, 0);
221         atomic_set(&hdev->cs_active_cnt, 0);
222
223         return 0;
224
225 free_eq_wq:
226         destroy_workqueue(hdev->eq_wq);
227 free_cq_wq:
228         destroy_workqueue(hdev->cq_wq);
229 asid_fini:
230         hl_asid_fini(hdev);
231 early_fini:
232         if (hdev->asic_funcs->early_fini)
233                 hdev->asic_funcs->early_fini(hdev);
234
235         return rc;
236 }
237
238 /*
239  * device_early_fini - finalize all that was done in device_early_init
240  *
241  * @hdev: pointer to habanalabs device structure
242  *
243  */
244 static void device_early_fini(struct hl_device *hdev)
245 {
246         mutex_destroy(&hdev->send_cpu_message_lock);
247
248         hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
249
250         kfree(hdev->hl_chip_info);
251
252         destroy_workqueue(hdev->eq_wq);
253         destroy_workqueue(hdev->cq_wq);
254
255         hl_asid_fini(hdev);
256
257         if (hdev->asic_funcs->early_fini)
258                 hdev->asic_funcs->early_fini(hdev);
259
260         mutex_destroy(&hdev->fd_open_cnt_lock);
261 }
262
263 static void set_freq_to_low_job(struct work_struct *work)
264 {
265         struct hl_device *hdev = container_of(work, struct hl_device,
266                                                 work_freq.work);
267
268         if (atomic_read(&hdev->fd_open_cnt) == 0)
269                 hl_device_set_frequency(hdev, PLL_LOW);
270
271         schedule_delayed_work(&hdev->work_freq,
272                         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
273 }
274
275 static void hl_device_heartbeat(struct work_struct *work)
276 {
277         struct hl_device *hdev = container_of(work, struct hl_device,
278                                                 work_heartbeat.work);
279
280         if (hl_device_disabled_or_in_reset(hdev))
281                 goto reschedule;
282
283         if (!hdev->asic_funcs->send_heartbeat(hdev))
284                 goto reschedule;
285
286         dev_err(hdev->dev, "Device heartbeat failed!\n");
287         hl_device_reset(hdev, true, false);
288
289         return;
290
291 reschedule:
292         schedule_delayed_work(&hdev->work_heartbeat,
293                         usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
294 }
295
296 /*
297  * device_late_init - do late stuff initialization for the habanalabs device
298  *
299  * @hdev: pointer to habanalabs device structure
300  *
301  * Do stuff that either needs the device H/W queues to be active or needs
302  * to happen after all the rest of the initialization is finished
303  */
304 static int device_late_init(struct hl_device *hdev)
305 {
306         int rc;
307
308         INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
309         hdev->high_pll = hdev->asic_prop.high_pll;
310
311         /* force setting to low frequency */
312         atomic_set(&hdev->curr_pll_profile, PLL_LOW);
313
314         if (hdev->pm_mng_profile == PM_AUTO)
315                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
316         else
317                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
318
319         if (hdev->asic_funcs->late_init) {
320                 rc = hdev->asic_funcs->late_init(hdev);
321                 if (rc) {
322                         dev_err(hdev->dev,
323                                 "failed late initialization for the H/W\n");
324                         return rc;
325                 }
326         }
327
328         schedule_delayed_work(&hdev->work_freq,
329                         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
330
331         if (hdev->heartbeat) {
332                 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
333                 schedule_delayed_work(&hdev->work_heartbeat,
334                                 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
335         }
336
337         hdev->late_init_done = true;
338
339         return 0;
340 }
341
342 /*
343  * device_late_fini - finalize all that was done in device_late_init
344  *
345  * @hdev: pointer to habanalabs device structure
346  *
347  */
348 static void device_late_fini(struct hl_device *hdev)
349 {
350         if (!hdev->late_init_done)
351                 return;
352
353         cancel_delayed_work_sync(&hdev->work_freq);
354         if (hdev->heartbeat)
355                 cancel_delayed_work_sync(&hdev->work_heartbeat);
356
357         if (hdev->asic_funcs->late_fini)
358                 hdev->asic_funcs->late_fini(hdev);
359
360         hdev->late_init_done = false;
361 }
362
363 /*
364  * hl_device_set_frequency - set the frequency of the device
365  *
366  * @hdev: pointer to habanalabs device structure
367  * @freq: the new frequency value
368  *
369  * Change the frequency if needed.
370  * We allose to set PLL to low only if there is no user process
371  * Returns 0 if no change was done, otherwise returns 1;
372  */
373 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
374 {
375         enum hl_pll_frequency old_freq =
376                         (freq == PLL_HIGH) ? PLL_LOW : PLL_HIGH;
377         int ret;
378
379         if (hdev->pm_mng_profile == PM_MANUAL)
380                 return 0;
381
382         ret = atomic_cmpxchg(&hdev->curr_pll_profile, old_freq, freq);
383         if (ret == freq)
384                 return 0;
385
386         /*
387          * in case we want to lower frequency, check if device is not
388          * opened. We must have a check here to workaround race condition with
389          * hl_device_open
390          */
391         if ((freq == PLL_LOW) && (atomic_read(&hdev->fd_open_cnt) > 0)) {
392                 atomic_set(&hdev->curr_pll_profile, PLL_HIGH);
393                 return 0;
394         }
395
396         dev_dbg(hdev->dev, "Changing device frequency to %s\n",
397                 freq == PLL_HIGH ? "high" : "low");
398
399         hdev->asic_funcs->set_pll_profile(hdev, freq);
400
401         return 1;
402 }
403
404 /*
405  * hl_device_suspend - initiate device suspend
406  *
407  * @hdev: pointer to habanalabs device structure
408  *
409  * Puts the hw in the suspend state (all asics).
410  * Returns 0 for success or an error on failure.
411  * Called at driver suspend.
412  */
413 int hl_device_suspend(struct hl_device *hdev)
414 {
415         int rc;
416
417         pci_save_state(hdev->pdev);
418
419         /* Block future CS/VM/JOB completion operations */
420         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
421         if (rc) {
422                 dev_err(hdev->dev, "Can't suspend while in reset\n");
423                 return -EIO;
424         }
425
426         /* This blocks all other stuff that is not blocked by in_reset */
427         hdev->disabled = true;
428
429         /*
430          * Flush anyone that is inside the critical section of enqueue
431          * jobs to the H/W
432          */
433         hdev->asic_funcs->hw_queues_lock(hdev);
434         hdev->asic_funcs->hw_queues_unlock(hdev);
435
436         /* Flush processes that are sending message to CPU */
437         mutex_lock(&hdev->send_cpu_message_lock);
438         mutex_unlock(&hdev->send_cpu_message_lock);
439
440         rc = hdev->asic_funcs->suspend(hdev);
441         if (rc)
442                 dev_err(hdev->dev,
443                         "Failed to disable PCI access of device CPU\n");
444
445         /* Shut down the device */
446         pci_disable_device(hdev->pdev);
447         pci_set_power_state(hdev->pdev, PCI_D3hot);
448
449         return 0;
450 }
451
452 /*
453  * hl_device_resume - initiate device resume
454  *
455  * @hdev: pointer to habanalabs device structure
456  *
457  * Bring the hw back to operating state (all asics).
458  * Returns 0 for success or an error on failure.
459  * Called at driver resume.
460  */
461 int hl_device_resume(struct hl_device *hdev)
462 {
463         int rc;
464
465         pci_set_power_state(hdev->pdev, PCI_D0);
466         pci_restore_state(hdev->pdev);
467         rc = pci_enable_device_mem(hdev->pdev);
468         if (rc) {
469                 dev_err(hdev->dev,
470                         "Failed to enable PCI device in resume\n");
471                 return rc;
472         }
473
474         pci_set_master(hdev->pdev);
475
476         rc = hdev->asic_funcs->resume(hdev);
477         if (rc) {
478                 dev_err(hdev->dev, "Failed to resume device after suspend\n");
479                 goto disable_device;
480         }
481
482
483         hdev->disabled = false;
484         atomic_set(&hdev->in_reset, 0);
485
486         rc = hl_device_reset(hdev, true, false);
487         if (rc) {
488                 dev_err(hdev->dev, "Failed to reset device during resume\n");
489                 goto disable_device;
490         }
491
492         return 0;
493
494 disable_device:
495         pci_clear_master(hdev->pdev);
496         pci_disable_device(hdev->pdev);
497
498         return rc;
499 }
500
501 static void hl_device_hard_reset_pending(struct work_struct *work)
502 {
503         struct hl_device_reset_work *device_reset_work =
504                 container_of(work, struct hl_device_reset_work, reset_work);
505         struct hl_device *hdev = device_reset_work->hdev;
506         u16 pending_total, pending_cnt;
507         struct task_struct *task = NULL;
508
509         if (hdev->pldm)
510                 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
511         else
512                 pending_total = HL_PENDING_RESET_PER_SEC;
513
514         pending_cnt = pending_total;
515
516         /* Flush all processes that are inside hl_open */
517         mutex_lock(&hdev->fd_open_cnt_lock);
518
519         while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
520
521                 pending_cnt--;
522
523                 dev_info(hdev->dev,
524                         "Can't HARD reset, waiting for user to close FD\n");
525                 ssleep(1);
526         }
527
528         if (atomic_read(&hdev->fd_open_cnt)) {
529                 task = get_pid_task(hdev->user_ctx->hpriv->taskpid,
530                                         PIDTYPE_PID);
531                 if (task) {
532                         dev_info(hdev->dev, "Killing user processes\n");
533                         send_sig(SIGKILL, task, 1);
534                         msleep(100);
535
536                         put_task_struct(task);
537                 }
538         }
539
540         pending_cnt = pending_total;
541
542         while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
543
544                 pending_cnt--;
545
546                 ssleep(1);
547         }
548
549         if (atomic_read(&hdev->fd_open_cnt))
550                 dev_crit(hdev->dev,
551                         "Going to hard reset with open user contexts\n");
552
553         mutex_unlock(&hdev->fd_open_cnt_lock);
554
555         hl_device_reset(hdev, true, true);
556
557         kfree(device_reset_work);
558 }
559
560 /*
561  * hl_device_reset - reset the device
562  *
563  * @hdev: pointer to habanalabs device structure
564  * @hard_reset: should we do hard reset to all engines or just reset the
565  *              compute/dma engines
566  *
567  * Block future CS and wait for pending CS to be enqueued
568  * Call ASIC H/W fini
569  * Flush all completions
570  * Re-initialize all internal data structures
571  * Call ASIC H/W init, late_init
572  * Test queues
573  * Enable device
574  *
575  * Returns 0 for success or an error on failure.
576  */
577 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
578                         bool from_hard_reset_thread)
579 {
580         int i, rc;
581
582         if (!hdev->init_done) {
583                 dev_err(hdev->dev,
584                         "Can't reset before initialization is done\n");
585                 return 0;
586         }
587
588         /*
589          * Prevent concurrency in this function - only one reset should be
590          * done at any given time. Only need to perform this if we didn't
591          * get from the dedicated hard reset thread
592          */
593         if (!from_hard_reset_thread) {
594                 /* Block future CS/VM/JOB completion operations */
595                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
596                 if (rc)
597                         return 0;
598
599                 /* This also blocks future CS/VM/JOB completion operations */
600                 hdev->disabled = true;
601
602                 /*
603                  * Flush anyone that is inside the critical section of enqueue
604                  * jobs to the H/W
605                  */
606                 hdev->asic_funcs->hw_queues_lock(hdev);
607                 hdev->asic_funcs->hw_queues_unlock(hdev);
608
609                 dev_err(hdev->dev, "Going to RESET device!\n");
610         }
611
612 again:
613         if ((hard_reset) && (!from_hard_reset_thread)) {
614                 struct hl_device_reset_work *device_reset_work;
615
616                 if (!hdev->pdev) {
617                         dev_err(hdev->dev,
618                                 "Reset action is NOT supported in simulator\n");
619                         rc = -EINVAL;
620                         goto out_err;
621                 }
622
623                 hdev->hard_reset_pending = true;
624
625                 device_reset_work = kzalloc(sizeof(*device_reset_work),
626                                                 GFP_ATOMIC);
627                 if (!device_reset_work) {
628                         rc = -ENOMEM;
629                         goto out_err;
630                 }
631
632                 /*
633                  * Because the reset function can't run from interrupt or
634                  * from heartbeat work, we need to call the reset function
635                  * from a dedicated work
636                  */
637                 INIT_WORK(&device_reset_work->reset_work,
638                                 hl_device_hard_reset_pending);
639                 device_reset_work->hdev = hdev;
640                 schedule_work(&device_reset_work->reset_work);
641
642                 return 0;
643         }
644
645         if (hard_reset) {
646                 device_late_fini(hdev);
647
648                 /*
649                  * Now that the heartbeat thread is closed, flush processes
650                  * which are sending messages to CPU
651                  */
652                 mutex_lock(&hdev->send_cpu_message_lock);
653                 mutex_unlock(&hdev->send_cpu_message_lock);
654         }
655
656         /*
657          * Halt the engines and disable interrupts so we won't get any more
658          * completions from H/W and we won't have any accesses from the
659          * H/W to the host machine
660          */
661         hdev->asic_funcs->halt_engines(hdev, hard_reset);
662
663         /* Go over all the queues, release all CS and their jobs */
664         hl_cs_rollback_all(hdev);
665
666         if (hard_reset) {
667                 /* Release kernel context */
668                 if (hl_ctx_put(hdev->kernel_ctx) != 1) {
669                         dev_err(hdev->dev,
670                                 "kernel ctx is alive during hard reset\n");
671                         rc = -EBUSY;
672                         goto out_err;
673                 }
674
675                 hdev->kernel_ctx = NULL;
676         }
677
678         /* Reset the H/W. It will be in idle state after this returns */
679         hdev->asic_funcs->hw_fini(hdev, hard_reset);
680
681         if (hard_reset) {
682                 hl_vm_fini(hdev);
683                 hl_eq_reset(hdev, &hdev->event_queue);
684         }
685
686         /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
687         hl_hw_queue_reset(hdev, hard_reset);
688         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
689                 hl_cq_reset(hdev, &hdev->completion_queue[i]);
690
691         /* Make sure the setup phase for the user context will run again */
692         if (hdev->user_ctx) {
693                 atomic_set(&hdev->user_ctx->thread_restore_token, 1);
694                 hdev->user_ctx->thread_restore_wait_token = 0;
695         }
696
697         /* Finished tear-down, starting to re-initialize */
698
699         if (hard_reset) {
700                 hdev->device_cpu_disabled = false;
701
702                 /* Allocate the kernel context */
703                 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
704                                                 GFP_KERNEL);
705                 if (!hdev->kernel_ctx) {
706                         rc = -ENOMEM;
707                         goto out_err;
708                 }
709
710                 hdev->user_ctx = NULL;
711
712                 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
713                 if (rc) {
714                         dev_err(hdev->dev,
715                                 "failed to init kernel ctx in hard reset\n");
716                         kfree(hdev->kernel_ctx);
717                         hdev->kernel_ctx = NULL;
718                         goto out_err;
719                 }
720         }
721
722         rc = hdev->asic_funcs->hw_init(hdev);
723         if (rc) {
724                 dev_err(hdev->dev,
725                         "failed to initialize the H/W after reset\n");
726                 goto out_err;
727         }
728
729         hdev->disabled = false;
730
731         /* Check that the communication with the device is working */
732         rc = hdev->asic_funcs->test_queues(hdev);
733         if (rc) {
734                 dev_err(hdev->dev,
735                         "Failed to detect if device is alive after reset\n");
736                 goto out_err;
737         }
738
739         if (hard_reset) {
740                 rc = device_late_init(hdev);
741                 if (rc) {
742                         dev_err(hdev->dev,
743                                 "Failed late init after hard reset\n");
744                         goto out_err;
745                 }
746
747                 rc = hl_vm_init(hdev);
748                 if (rc) {
749                         dev_err(hdev->dev,
750                                 "Failed to init memory module after hard reset\n");
751                         goto out_err;
752                 }
753
754                 hl_set_max_power(hdev, hdev->max_power);
755
756                 hdev->hard_reset_pending = false;
757         } else {
758                 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
759                 if (rc) {
760                         dev_err(hdev->dev,
761                                 "Failed late init after soft reset\n");
762                         goto out_err;
763                 }
764         }
765
766         atomic_set(&hdev->in_reset, 0);
767
768         if (hard_reset)
769                 hdev->hard_reset_cnt++;
770         else
771                 hdev->soft_reset_cnt++;
772
773         return 0;
774
775 out_err:
776         hdev->disabled = true;
777
778         if (hard_reset) {
779                 dev_err(hdev->dev,
780                         "Failed to reset! Device is NOT usable\n");
781                 hdev->hard_reset_cnt++;
782         } else {
783                 dev_err(hdev->dev,
784                         "Failed to do soft-reset, trying hard reset\n");
785                 hdev->soft_reset_cnt++;
786                 hard_reset = true;
787                 goto again;
788         }
789
790         atomic_set(&hdev->in_reset, 0);
791
792         return rc;
793 }
794
795 /*
796  * hl_device_init - main initialization function for habanalabs device
797  *
798  * @hdev: pointer to habanalabs device structure
799  *
800  * Allocate an id for the device, do early initialization and then call the
801  * ASIC specific initialization functions. Finally, create the cdev and the
802  * Linux device to expose it to the user
803  */
804 int hl_device_init(struct hl_device *hdev, struct class *hclass)
805 {
806         int i, rc, cq_ready_cnt;
807
808         /* Create device */
809         rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);
810
811         if (rc)
812                 goto out_disabled;
813
814         /* Initialize ASIC function pointers and perform early init */
815         rc = device_early_init(hdev);
816         if (rc)
817                 goto release_device;
818
819         /*
820          * Start calling ASIC initialization. First S/W then H/W and finally
821          * late init
822          */
823         rc = hdev->asic_funcs->sw_init(hdev);
824         if (rc)
825                 goto early_fini;
826
827         /*
828          * Initialize the H/W queues. Must be done before hw_init, because
829          * there the addresses of the kernel queue are being written to the
830          * registers of the device
831          */
832         rc = hl_hw_queues_create(hdev);
833         if (rc) {
834                 dev_err(hdev->dev, "failed to initialize kernel queues\n");
835                 goto sw_fini;
836         }
837
838         /*
839          * Initialize the completion queues. Must be done before hw_init,
840          * because there the addresses of the completion queues are being
841          * passed as arguments to request_irq
842          */
843         hdev->completion_queue =
844                         kcalloc(hdev->asic_prop.completion_queues_count,
845                                 sizeof(*hdev->completion_queue), GFP_KERNEL);
846
847         if (!hdev->completion_queue) {
848                 dev_err(hdev->dev, "failed to allocate completion queues\n");
849                 rc = -ENOMEM;
850                 goto hw_queues_destroy;
851         }
852
853         for (i = 0, cq_ready_cnt = 0;
854                         i < hdev->asic_prop.completion_queues_count;
855                         i++, cq_ready_cnt++) {
856                 rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
857                 if (rc) {
858                         dev_err(hdev->dev,
859                                 "failed to initialize completion queue\n");
860                         goto cq_fini;
861                 }
862         }
863
864         /*
865          * Initialize the event queue. Must be done before hw_init,
866          * because there the address of the event queue is being
867          * passed as argument to request_irq
868          */
869         rc = hl_eq_init(hdev, &hdev->event_queue);
870         if (rc) {
871                 dev_err(hdev->dev, "failed to initialize event queue\n");
872                 goto cq_fini;
873         }
874
875         /* Allocate the kernel context */
876         hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
877         if (!hdev->kernel_ctx) {
878                 rc = -ENOMEM;
879                 goto eq_fini;
880         }
881
882         hdev->user_ctx = NULL;
883
884         rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
885         if (rc) {
886                 dev_err(hdev->dev, "failed to initialize kernel context\n");
887                 goto free_ctx;
888         }
889
890         rc = hl_cb_pool_init(hdev);
891         if (rc) {
892                 dev_err(hdev->dev, "failed to initialize CB pool\n");
893                 goto release_ctx;
894         }
895
896         rc = hl_sysfs_init(hdev);
897         if (rc) {
898                 dev_err(hdev->dev, "failed to initialize sysfs\n");
899                 goto free_cb_pool;
900         }
901
902         hl_debugfs_add_device(hdev);
903
904         if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
905                 dev_info(hdev->dev,
906                         "H/W state is dirty, must reset before initializing\n");
907                 hdev->asic_funcs->hw_fini(hdev, true);
908         }
909
910         rc = hdev->asic_funcs->hw_init(hdev);
911         if (rc) {
912                 dev_err(hdev->dev, "failed to initialize the H/W\n");
913                 rc = 0;
914                 goto out_disabled;
915         }
916
917         hdev->disabled = false;
918
919         /* Check that the communication with the device is working */
920         rc = hdev->asic_funcs->test_queues(hdev);
921         if (rc) {
922                 dev_err(hdev->dev, "Failed to detect if device is alive\n");
923                 rc = 0;
924                 goto out_disabled;
925         }
926
927         /* After test_queues, KMD can start sending messages to device CPU */
928
929         rc = device_late_init(hdev);
930         if (rc) {
931                 dev_err(hdev->dev, "Failed late initialization\n");
932                 rc = 0;
933                 goto out_disabled;
934         }
935
936         dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
937                 hdev->asic_name,
938                 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
939
940         rc = hl_vm_init(hdev);
941         if (rc) {
942                 dev_err(hdev->dev, "Failed to initialize memory module\n");
943                 rc = 0;
944                 goto out_disabled;
945         }
946
947         /*
948          * hl_hwmon_init must be called after device_late_init, because only
949          * there we get the information from the device about which
950          * hwmon-related sensors the device supports
951          */
952         rc = hl_hwmon_init(hdev);
953         if (rc) {
954                 dev_err(hdev->dev, "Failed to initialize hwmon\n");
955                 rc = 0;
956                 goto out_disabled;
957         }
958
959         dev_notice(hdev->dev,
960                 "Successfully added device to habanalabs driver\n");
961
962         hdev->init_done = true;
963
964         return 0;
965
966 free_cb_pool:
967         hl_cb_pool_fini(hdev);
968 release_ctx:
969         if (hl_ctx_put(hdev->kernel_ctx) != 1)
970                 dev_err(hdev->dev,
971                         "kernel ctx is still alive on initialization failure\n");
972 free_ctx:
973         kfree(hdev->kernel_ctx);
974 eq_fini:
975         hl_eq_fini(hdev, &hdev->event_queue);
976 cq_fini:
977         for (i = 0 ; i < cq_ready_cnt ; i++)
978                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
979         kfree(hdev->completion_queue);
980 hw_queues_destroy:
981         hl_hw_queues_destroy(hdev);
982 sw_fini:
983         hdev->asic_funcs->sw_fini(hdev);
984 early_fini:
985         device_early_fini(hdev);
986 release_device:
987         device_destroy(hclass, hdev->dev->devt);
988         cdev_del(&hdev->cdev);
989 out_disabled:
990         hdev->disabled = true;
991         if (hdev->pdev)
992                 dev_err(&hdev->pdev->dev,
993                         "Failed to initialize hl%d. Device is NOT usable !\n",
994                         hdev->id);
995         else
996                 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
997                         hdev->id);
998
999         return rc;
1000 }
1001
1002 /*
1003  * hl_device_fini - main tear-down function for habanalabs device
1004  *
1005  * @hdev: pointer to habanalabs device structure
1006  *
1007  * Destroy the device, call ASIC fini functions and release the id
1008  */
1009 void hl_device_fini(struct hl_device *hdev)
1010 {
1011         int i, rc;
1012         ktime_t timeout;
1013
1014         dev_info(hdev->dev, "Removing device\n");
1015
1016         /*
1017          * This function is competing with the reset function, so try to
1018          * take the reset atomic and if we are already in middle of reset,
1019          * wait until reset function is finished. Reset function is designed
1020          * to always finish (could take up to a few seconds in worst case).
1021          */
1022
1023         timeout = ktime_add_us(ktime_get(),
1024                                 HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
1025         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1026         while (rc) {
1027                 usleep_range(50, 200);
1028                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1029                 if (ktime_compare(ktime_get(), timeout) > 0) {
1030                         WARN(1, "Failed to remove device because reset function did not finish\n");
1031                         return;
1032                 }
1033         };
1034
1035         /* Mark device as disabled */
1036         hdev->disabled = true;
1037
1038         hl_hwmon_fini(hdev);
1039
1040         device_late_fini(hdev);
1041
1042         hl_debugfs_remove_device(hdev);
1043
1044         hl_sysfs_fini(hdev);
1045
1046         /*
1047          * Halt the engines and disable interrupts so we won't get any more
1048          * completions from H/W and we won't have any accesses from the
1049          * H/W to the host machine
1050          */
1051         hdev->asic_funcs->halt_engines(hdev, true);
1052
1053         /* Go over all the queues, release all CS and their jobs */
1054         hl_cs_rollback_all(hdev);
1055
1056         hl_cb_pool_fini(hdev);
1057
1058         /* Release kernel context */
1059         if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1060                 dev_err(hdev->dev, "kernel ctx is still alive\n");
1061
1062         /* Reset the H/W. It will be in idle state after this returns */
1063         hdev->asic_funcs->hw_fini(hdev, true);
1064
1065         hl_vm_fini(hdev);
1066
1067         hl_eq_fini(hdev, &hdev->event_queue);
1068
1069         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1070                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1071         kfree(hdev->completion_queue);
1072
1073         hl_hw_queues_destroy(hdev);
1074
1075         /* Call ASIC S/W finalize function */
1076         hdev->asic_funcs->sw_fini(hdev);
1077
1078         device_early_fini(hdev);
1079
1080         /* Hide device from user */
1081         device_destroy(hdev->dev->class, hdev->dev->devt);
1082         cdev_del(&hdev->cdev);
1083
1084         pr_info("removed device successfully\n");
1085 }
1086
1087 /*
1088  * hl_poll_timeout_memory - Periodically poll a host memory address
1089  *                              until it is not zero or a timeout occurs
1090  * @hdev: pointer to habanalabs device structure
1091  * @addr: Address to poll
1092  * @timeout_us: timeout in us
1093  * @val: Variable to read the value into
1094  *
1095  * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
1096  * case, the last read value at @addr is stored in @val. Must not
1097  * be called from atomic context if sleep_us or timeout_us are used.
1098  *
1099  * The function sleeps for 100us with timeout value of
1100  * timeout_us
1101  */
1102 int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr,
1103                                 u32 timeout_us, u32 *val)
1104 {
1105         /*
1106          * address in this function points always to a memory location in the
1107          * host's (server's) memory. That location is updated asynchronously
1108          * either by the direct access of the device or by another core
1109          */
1110         u32 *paddr = (u32 *) (uintptr_t) addr;
1111         ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
1112
1113         might_sleep();
1114
1115         for (;;) {
1116                 /*
1117                  * Flush CPU read/write buffers to make sure we read updates
1118                  * done by other cores or by the device
1119                  */
1120                 mb();
1121                 *val = *paddr;
1122                 if (*val)
1123                         break;
1124                 if (ktime_compare(ktime_get(), timeout) > 0) {
1125                         *val = *paddr;
1126                         break;
1127                 }
1128                 usleep_range((100 >> 2) + 1, 100);
1129         }
1130
1131         return *val ? 0 : -ETIMEDOUT;
1132 }
1133
1134 /*
1135  * hl_poll_timeout_devicememory - Periodically poll a device memory address
1136  *                                until it is not zero or a timeout occurs
1137  * @hdev: pointer to habanalabs device structure
1138  * @addr: Device address to poll
1139  * @timeout_us: timeout in us
1140  * @val: Variable to read the value into
1141  *
1142  * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
1143  * case, the last read value at @addr is stored in @val. Must not
1144  * be called from atomic context if sleep_us or timeout_us are used.
1145  *
1146  * The function sleeps for 100us with timeout value of
1147  * timeout_us
1148  */
1149 int hl_poll_timeout_device_memory(struct hl_device *hdev, void __iomem *addr,
1150                                 u32 timeout_us, u32 *val)
1151 {
1152         ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
1153
1154         might_sleep();
1155
1156         for (;;) {
1157                 *val = readl(addr);
1158                 if (*val)
1159                         break;
1160                 if (ktime_compare(ktime_get(), timeout) > 0) {
1161                         *val = readl(addr);
1162                         break;
1163                 }
1164                 usleep_range((100 >> 2) + 1, 100);
1165         }
1166
1167         return *val ? 0 : -ETIMEDOUT;
1168 }
1169
1170 /*
1171  * MMIO register access helper functions.
1172  */
1173
1174 /*
1175  * hl_rreg - Read an MMIO register
1176  *
1177  * @hdev: pointer to habanalabs device structure
1178  * @reg: MMIO register offset (in bytes)
1179  *
1180  * Returns the value of the MMIO register we are asked to read
1181  *
1182  */
1183 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1184 {
1185         return readl(hdev->rmmio + reg);
1186 }
1187
1188 /*
1189  * hl_wreg - Write to an MMIO register
1190  *
1191  * @hdev: pointer to habanalabs device structure
1192  * @reg: MMIO register offset (in bytes)
1193  * @val: 32-bit value
1194  *
1195  * Writes the 32-bit value into the MMIO register
1196  *
1197  */
1198 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1199 {
1200         writel(val, hdev->rmmio + reg);
1201 }