Subversion Repositories Kolibri OS

Rev

Rev 6935 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * drm_irq.c IRQ and vblank support
  3.  *
  4.  * \author Rickard E. (Rik) Faith <faith@valinux.com>
  5.  * \author Gareth Hughes <gareth@valinux.com>
  6.  */
  7.  
  8. /*
  9.  * Created: Fri Mar 19 14:30:16 1999 by faith@valinux.com
  10.  *
  11.  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  12.  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  13.  * All Rights Reserved.
  14.  *
  15.  * Permission is hereby granted, free of charge, to any person obtaining a
  16.  * copy of this software and associated documentation files (the "Software"),
  17.  * to deal in the Software without restriction, including without limitation
  18.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  19.  * and/or sell copies of the Software, and to permit persons to whom the
  20.  * Software is furnished to do so, subject to the following conditions:
  21.  *
  22.  * The above copyright notice and this permission notice (including the next
  23.  * paragraph) shall be included in all copies or substantial portions of the
  24.  * Software.
  25.  *
  26.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  27.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  28.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  29.  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  30.  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  31.  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  32.  * OTHER DEALINGS IN THE SOFTWARE.
  33.  */
  34.  
  35. #include <drm/drmP.h>
  36. //#include "drm_trace.h"
  37. #include "drm_internal.h"
  38.  
  39. #include <linux/interrupt.h>    /* For task queue support */
  40. #include <linux/slab.h>
  41.  
  42. #include <linux/vgaarb.h>
  43. #include <linux/export.h>
  44.  
  45. ktime_t ktime_get(void);
  46.  
  47. static inline ktime_t ktime_get_real(void)
  48. {
  49.         return ktime_get();
  50. }
  51.  
  52. static inline ktime_t ktime_mono_to_real(ktime_t mono)
  53. {
  54.         return mono;
  55. }
  56.  
  57. irqreturn_t device_irq_handler(struct drm_device *dev)
  58. {
  59.         return dev->driver->irq_handler(0, dev);
  60. }
  61.  
  62. /* Access macro for slots in vblank timestamp ringbuffer. */
  63. #define vblanktimestamp(dev, pipe, count) \
  64.         ((dev)->vblank[pipe].time[(count) % DRM_VBLANKTIME_RBSIZE])
  65.  
  66. /* Retry timestamp calculation up to 3 times to satisfy
  67.  * drm_timestamp_precision before giving up.
  68.  */
  69. #define DRM_TIMESTAMP_MAXRETRIES 3
  70.  
  71. /* Threshold in nanoseconds for detection of redundant
  72.  * vblank irq in drm_handle_vblank(). 1 msec should be ok.
  73.  */
  74. #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
  75.  
  76. static bool
  77. drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  78.                           struct timeval *tvblank, unsigned flags);
  79.  
  80. static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
  81.  
  82. /*
  83.  * Default to use monotonic timestamps for wait-for-vblank and page-flip
  84.  * complete events.
  85.  */
  86. unsigned int drm_timestamp_monotonic = 1;
  87.  
  88. static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
  89.  
  90. module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
  91. module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
  92. module_param_named(timestamp_monotonic, drm_timestamp_monotonic, int, 0600);
  93.  
  94. static void store_vblank(struct drm_device *dev, unsigned int pipe,
  95.                          u32 vblank_count_inc,
  96.                          struct timeval *t_vblank, u32 last)
  97. {
  98.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  99.         u32 tslot;
  100.  
  101.         assert_spin_locked(&dev->vblank_time_lock);
  102.  
  103.         vblank->last = last;
  104.  
  105.         /* All writers hold the spinlock, but readers are serialized by
  106.          * the latching of vblank->count below.
  107.          */
  108.         tslot = vblank->count + vblank_count_inc;
  109.         vblanktimestamp(dev, pipe, tslot) = *t_vblank;
  110.  
  111.         /*
  112.          * vblank timestamp updates are protected on the write side with
  113.          * vblank_time_lock, but on the read side done locklessly using a
  114.          * sequence-lock on the vblank counter. Ensure correct ordering using
  115.          * memory barrriers. We need the barrier both before and also after the
  116.          * counter update to synchronize with the next timestamp write.
  117.          * The read-side barriers for this are in drm_vblank_count_and_time.
  118.          */
  119.         smp_wmb();
  120.         vblank->count += vblank_count_inc;
  121.         smp_wmb();
  122. }
  123.  
  124. /**
  125.  * drm_reset_vblank_timestamp - reset the last timestamp to the last vblank
  126.  * @dev: DRM device
  127.  * @pipe: index of CRTC for which to reset the timestamp
  128.  *
  129.  * Reset the stored timestamp for the current vblank count to correspond
  130.  * to the last vblank occurred.
  131.  *
  132.  * Only to be called from drm_vblank_on().
  133.  *
  134.  * Note: caller must hold dev->vbl_lock since this reads & writes
  135.  * device vblank fields.
  136.  */
  137. static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
  138. {
  139.         u32 cur_vblank;
  140.         bool rc;
  141.         struct timeval t_vblank;
  142.         int count = DRM_TIMESTAMP_MAXRETRIES;
  143.  
  144.         spin_lock(&dev->vblank_time_lock);
  145.  
  146.         /*
  147.          * sample the current counter to avoid random jumps
  148.          * when drm_vblank_enable() applies the diff
  149.          */
  150.         do {
  151.                 cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
  152.                 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, 0);
  153.         } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
  154.  
  155.         /*
  156.          * Only reinitialize corresponding vblank timestamp if high-precision query
  157.          * available and didn't fail. Otherwise reinitialize delayed at next vblank
  158.          * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
  159.          */
  160.         if (!rc)
  161.                 t_vblank = (struct timeval) {0, 0};
  162.  
  163.         /*
  164.          * +1 to make sure user will never see the same
  165.          * vblank counter value before and after a modeset
  166.          */
  167.         store_vblank(dev, pipe, 1, &t_vblank, cur_vblank);
  168.  
  169.         spin_unlock(&dev->vblank_time_lock);
  170. }
  171.  
  172. /**
  173.  * drm_update_vblank_count - update the master vblank counter
  174.  * @dev: DRM device
  175.  * @pipe: counter to update
  176.  *
  177.  * Call back into the driver to update the appropriate vblank counter
  178.  * (specified by @pipe).  Deal with wraparound, if it occurred, and
  179.  * update the last read value so we can deal with wraparound on the next
  180.  * call if necessary.
  181.  *
  182.  * Only necessary when going from off->on, to account for frames we
  183.  * didn't get an interrupt for.
  184.  *
  185.  * Note: caller must hold dev->vbl_lock since this reads & writes
  186.  * device vblank fields.
  187.  */
  188. static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
  189.                                     unsigned long flags)
  190. {
  191.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  192.         u32 cur_vblank, diff;
  193.         bool rc;
  194.         struct timeval t_vblank;
  195.         int count = DRM_TIMESTAMP_MAXRETRIES;
  196.         int framedur_ns = vblank->framedur_ns;
  197.  
  198.         /*
  199.          * Interrupts were disabled prior to this call, so deal with counter
  200.          * wrap if needed.
  201.          * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
  202.          * here if the register is small or we had vblank interrupts off for
  203.          * a long time.
  204.          *
  205.          * We repeat the hardware vblank counter & timestamp query until
  206.          * we get consistent results. This to prevent races between gpu
  207.          * updating its hardware counter while we are retrieving the
  208.          * corresponding vblank timestamp.
  209.          */
  210.         do {
  211.                 cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
  212.                 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, flags);
  213.         } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
  214.  
  215.         if (dev->max_vblank_count != 0) {
  216.                 /* trust the hw counter when it's around */
  217.                 diff = (cur_vblank - vblank->last) & dev->max_vblank_count;
  218.         } else if (rc && framedur_ns) {
  219.                 const struct timeval *t_old;
  220.                 u64 diff_ns;
  221.  
  222.                 t_old = &vblanktimestamp(dev, pipe, vblank->count);
  223.                 diff_ns = timeval_to_ns(&t_vblank) - timeval_to_ns(t_old);
  224.  
  225.                 /*
  226.                  * Figure out how many vblanks we've missed based
  227.                  * on the difference in the timestamps and the
  228.                  * frame/field duration.
  229.                  */
  230.                 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
  231.  
  232.                 if (diff == 0 && flags & DRM_CALLED_FROM_VBLIRQ)
  233.                         DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored."
  234.                                       " diff_ns = %lld, framedur_ns = %d)\n",
  235.                                       pipe, (long long) diff_ns, framedur_ns);
  236.         } else {
  237.                 /* some kind of default for drivers w/o accurate vbl timestamping */
  238.                 diff = (flags & DRM_CALLED_FROM_VBLIRQ) != 0;
  239.         }
  240.  
  241.         /*
  242.          * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
  243.          * interval? If so then vblank irqs keep running and it will likely
  244.          * happen that the hardware vblank counter is not trustworthy as it
  245.          * might reset at some point in that interval and vblank timestamps
  246.          * are not trustworthy either in that interval. Iow. this can result
  247.          * in a bogus diff >> 1 which must be avoided as it would cause
  248.          * random large forward jumps of the software vblank counter.
  249.          */
  250.         if (diff > 1 && (vblank->inmodeset & 0x2)) {
  251.                 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
  252.                               " due to pre-modeset.\n", pipe, diff);
  253.                 diff = 1;
  254.         }
  255.  
  256.         /*
  257.          * FIMXE: Need to replace this hack with proper seqlocks.
  258.          *
  259.          * Restrict the bump of the software vblank counter to a safe maximum
  260.          * value of +1 whenever there is the possibility that concurrent readers
  261.          * of vblank timestamps could be active at the moment, as the current
  262.          * implementation of the timestamp caching and updating is not safe
  263.          * against concurrent readers for calls to store_vblank() with a bump
  264.          * of anything but +1. A bump != 1 would very likely return corrupted
  265.          * timestamps to userspace, because the same slot in the cache could
  266.          * be concurrently written by store_vblank() and read by one of those
  267.          * readers without the read-retry logic detecting the collision.
  268.          *
  269.          * Concurrent readers can exist when we are called from the
  270.          * drm_vblank_off() or drm_vblank_on() functions and other non-vblank-
  271.          * irq callers. However, all those calls to us are happening with the
  272.          * vbl_lock locked to prevent drm_vblank_get(), so the vblank refcount
  273.          * can't increase while we are executing. Therefore a zero refcount at
  274.          * this point is safe for arbitrary counter bumps if we are called
  275.          * outside vblank irq, a non-zero count is not 100% safe. Unfortunately
  276.          * we must also accept a refcount of 1, as whenever we are called from
  277.          * drm_vblank_get() -> drm_vblank_enable() the refcount will be 1 and
  278.          * we must let that one pass through in order to not lose vblank counts
  279.          * during vblank irq off - which would completely defeat the whole
  280.          * point of this routine.
  281.          *
  282.          * Whenever we are called from vblank irq, we have to assume concurrent
  283.          * readers exist or can show up any time during our execution, even if
  284.          * the refcount is currently zero, as vblank irqs are usually only
  285.          * enabled due to the presence of readers, and because when we are called
  286.          * from vblank irq we can't hold the vbl_lock to protect us from sudden
  287.          * bumps in vblank refcount. Therefore also restrict bumps to +1 when
  288.          * called from vblank irq.
  289.          */
  290.         if ((diff > 1) && (atomic_read(&vblank->refcount) > 1 ||
  291.             (flags & DRM_CALLED_FROM_VBLIRQ))) {
  292.                 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u "
  293.                               "refcount %u, vblirq %u\n", pipe, diff,
  294.                               atomic_read(&vblank->refcount),
  295.                               (flags & DRM_CALLED_FROM_VBLIRQ) != 0);
  296.                 diff = 1;
  297.         }
  298.  
  299.         DRM_DEBUG_VBL("updating vblank count on crtc %u:"
  300.                       " current=%u, diff=%u, hw=%u hw_last=%u\n",
  301.                       pipe, vblank->count, diff, cur_vblank, vblank->last);
  302.  
  303.         if (diff == 0) {
  304.                 WARN_ON_ONCE(cur_vblank != vblank->last);
  305.                 return;
  306.         }
  307.  
  308.         /*
  309.          * Only reinitialize corresponding vblank timestamp if high-precision query
  310.          * available and didn't fail, or we were called from the vblank interrupt.
  311.          * Otherwise reinitialize delayed at next vblank interrupt and assign 0
  312.          * for now, to mark the vblanktimestamp as invalid.
  313.          */
  314.         if (!rc && (flags & DRM_CALLED_FROM_VBLIRQ) == 0)
  315.                 t_vblank = (struct timeval) {0, 0};
  316.  
  317.         store_vblank(dev, pipe, diff, &t_vblank, cur_vblank);
  318. }
  319.  
  320. /*
  321.  * Disable vblank irq's on crtc, make sure that last vblank count
  322.  * of hardware and corresponding consistent software vblank counter
  323.  * are preserved, even if there are any spurious vblank irq's after
  324.  * disable.
  325.  */
  326. static void vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
  327. {
  328.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  329.         unsigned long irqflags;
  330.  
  331.         /* Prevent vblank irq processing while disabling vblank irqs,
  332.          * so no updates of timestamps or count can happen after we've
  333.          * disabled. Needed to prevent races in case of delayed irq's.
  334.          */
  335.         spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
  336.  
  337.         /*
  338.          * Only disable vblank interrupts if they're enabled. This avoids
  339.          * calling the ->disable_vblank() operation in atomic context with the
  340.          * hardware potentially runtime suspended.
  341.          */
  342.         if (vblank->enabled) {
  343.                 dev->driver->disable_vblank(dev, pipe);
  344.                 vblank->enabled = false;
  345.         }
  346.  
  347.         /*
  348.          * Always update the count and timestamp to maintain the
  349.          * appearance that the counter has been ticking all along until
  350.          * this time. This makes the count account for the entire time
  351.          * between drm_vblank_on() and drm_vblank_off().
  352.          */
  353.         drm_update_vblank_count(dev, pipe, 0);
  354.  
  355.         spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
  356. }
  357.  
  358. static void vblank_disable_fn(unsigned long arg)
  359. {
  360.         struct drm_vblank_crtc *vblank = (void *)arg;
  361.         struct drm_device *dev = vblank->dev;
  362.         unsigned int pipe = vblank->pipe;
  363.         unsigned long irqflags;
  364.  
  365.         if (!dev->vblank_disable_allowed)
  366.                 return;
  367.  
  368.         spin_lock_irqsave(&dev->vbl_lock, irqflags);
  369.         if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
  370.                 DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
  371.                 vblank_disable_and_save(dev, pipe);
  372.         }
  373.         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  374. }
  375.  
  376. /**
  377.  * drm_vblank_cleanup - cleanup vblank support
  378.  * @dev: DRM device
  379.  *
  380.  * This function cleans up any resources allocated in drm_vblank_init.
  381.  */
  382. void drm_vblank_cleanup(struct drm_device *dev)
  383. {
  384.         unsigned int pipe;
  385.  
  386.         /* Bail if the driver didn't call drm_vblank_init() */
  387.         if (dev->num_crtcs == 0)
  388.                 return;
  389.  
  390.         for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
  391.                 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  392.  
  393.                 WARN_ON(vblank->enabled &&
  394.                         drm_core_check_feature(dev, DRIVER_MODESET));
  395.  
  396.                 del_timer_sync(&vblank->disable_timer);
  397.         }
  398.  
  399.         kfree(dev->vblank);
  400.  
  401.         dev->num_crtcs = 0;
  402. }
  403. EXPORT_SYMBOL(drm_vblank_cleanup);
  404.  
  405. /**
  406.  * drm_vblank_init - initialize vblank support
  407.  * @dev: DRM device
  408.  * @num_crtcs: number of CRTCs supported by @dev
  409.  *
  410.  * This function initializes vblank support for @num_crtcs display pipelines.
  411.  *
  412.  * Returns:
  413.  * Zero on success or a negative error code on failure.
  414.  */
  415. int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
  416. {
  417.         int ret = -ENOMEM;
  418.         unsigned int i;
  419.  
  420.         spin_lock_init(&dev->vbl_lock);
  421.         spin_lock_init(&dev->vblank_time_lock);
  422.  
  423.         dev->num_crtcs = num_crtcs;
  424.  
  425.         dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
  426.         if (!dev->vblank)
  427.                 goto err;
  428.  
  429.         for (i = 0; i < num_crtcs; i++) {
  430.                 struct drm_vblank_crtc *vblank = &dev->vblank[i];
  431.  
  432.                 vblank->dev = dev;
  433.                 vblank->pipe = i;
  434.                 init_waitqueue_head(&vblank->queue);
  435.                 setup_timer(&vblank->disable_timer, vblank_disable_fn,
  436.                             (unsigned long)vblank);
  437.         }
  438.  
  439.         DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
  440.  
  441.         /* Driver specific high-precision vblank timestamping supported? */
  442.         if (dev->driver->get_vblank_timestamp)
  443.                 DRM_INFO("Driver supports precise vblank timestamp query.\n");
  444.         else
  445.                 DRM_INFO("No driver support for vblank timestamp query.\n");
  446.  
  447.         /* Must have precise timestamping for reliable vblank instant disable */
  448.         if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
  449.                 dev->vblank_disable_immediate = false;
  450.                 DRM_INFO("Setting vblank_disable_immediate to false because "
  451.                          "get_vblank_timestamp == NULL\n");
  452.         }
  453.  
  454.         dev->vblank_disable_allowed = false;
  455.  
  456.         return 0;
  457.  
  458. err:
  459.         dev->num_crtcs = 0;
  460.         return ret;
  461. }
  462. EXPORT_SYMBOL(drm_vblank_init);
  463.  
  464.  
  465.  
  466.  
  467. /**
  468.  * drm_irq_install - install IRQ handler
  469.  * @dev: DRM device
  470.  * @irq: IRQ number to install the handler for
  471.  *
  472.  * Initializes the IRQ related data. Installs the handler, calling the driver
  473.  * irq_preinstall() and irq_postinstall() functions before and after the
  474.  * installation.
  475.  *
  476.  * This is the simplified helper interface provided for drivers with no special
  477.  * needs. Drivers which need to install interrupt handlers for multiple
  478.  * interrupts must instead set drm_device->irq_enabled to signal the DRM core
  479.  * that vblank interrupts are available.
  480.  *
  481.  * Returns:
  482.  * Zero on success or a negative error code on failure.
  483.  */
  484. int drm_irq_install(struct drm_device *dev, int irq)
  485. {
  486.         int ret;
  487.         unsigned long sh_flags = 0;
  488.  
  489.         if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
  490.                 return -EINVAL;
  491.  
  492.         if (irq == 0)
  493.                 return -EINVAL;
  494.  
  495.         /* Driver must have been initialized */
  496.         if (!dev->dev_private)
  497.                 return -EINVAL;
  498.  
  499.         if (dev->irq_enabled)
  500.                 return -EBUSY;
  501.         dev->irq_enabled = true;
  502.  
  503.         DRM_DEBUG("irq=%d\n", irq);
  504.  
  505.         /* Before installing handler */
  506.         if (dev->driver->irq_preinstall)
  507.                 dev->driver->irq_preinstall(dev);
  508.  
  509.     ret = !AttachIntHandler(irq, device_irq_handler, (u32)dev);
  510.  
  511.         /* After installing handler */
  512.         if (dev->driver->irq_postinstall)
  513.                 ret = dev->driver->irq_postinstall(dev);
  514.  
  515.         if (ret < 0) {
  516.                 dev->irq_enabled = false;
  517.         } else {
  518.                 dev->irq = irq;
  519.         }
  520.  
  521.     u16 cmd = PciRead16(dev->pdev->busnr, dev->pdev->devfn, 4);
  522.     cmd&= ~(1<<10);
  523.     PciWrite16(dev->pdev->busnr, dev->pdev->devfn, 4, cmd);
  524.  
  525.     return ret;
  526. }
  527. EXPORT_SYMBOL(drm_irq_install);
  528.  
  529.  
  530.  
  531.  
  532.  
  533. /**
  534.  * drm_calc_timestamping_constants - calculate vblank timestamp constants
  535.  * @crtc: drm_crtc whose timestamp constants should be updated.
  536.  * @mode: display mode containing the scanout timings
  537.  *
  538.  * Calculate and store various constants which are later
  539.  * needed by vblank and swap-completion timestamping, e.g,
  540.  * by drm_calc_vbltimestamp_from_scanoutpos(). They are
  541.  * derived from CRTC's true scanout timing, so they take
  542.  * things like panel scaling or other adjustments into account.
  543.  */
  544. void drm_calc_timestamping_constants(struct drm_crtc *crtc,
  545.                                      const struct drm_display_mode *mode)
  546. {
  547.         struct drm_device *dev = crtc->dev;
  548.         unsigned int pipe = drm_crtc_index(crtc);
  549.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  550.         int linedur_ns = 0, framedur_ns = 0;
  551.         int dotclock = mode->crtc_clock;
  552.  
  553.         if (!dev->num_crtcs)
  554.                 return;
  555.  
  556.         if (WARN_ON(pipe >= dev->num_crtcs))
  557.                 return;
  558.  
  559.         /* Valid dotclock? */
  560.         if (dotclock > 0) {
  561.                 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
  562.  
  563.                 /*
  564.                  * Convert scanline length in pixels and video
  565.                  * dot clock to line duration and frame duration
  566.                  * in nanoseconds:
  567.                  */
  568.                 linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
  569.                 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
  570.  
  571.                 /*
  572.                  * Fields of interlaced scanout modes are only half a frame duration.
  573.                  */
  574.                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
  575.                         framedur_ns /= 2;
  576.         } else
  577.                 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
  578.                           crtc->base.id);
  579.  
  580.         vblank->linedur_ns  = linedur_ns;
  581.         vblank->framedur_ns = framedur_ns;
  582.  
  583.         DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
  584.                   crtc->base.id, mode->crtc_htotal,
  585.                   mode->crtc_vtotal, mode->crtc_vdisplay);
  586.         DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
  587.                   crtc->base.id, dotclock, framedur_ns, linedur_ns);
  588. }
  589. EXPORT_SYMBOL(drm_calc_timestamping_constants);
  590.  
  591. /**
  592.  * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
  593.  * @dev: DRM device
  594.  * @pipe: index of CRTC whose vblank timestamp to retrieve
  595.  * @max_error: Desired maximum allowable error in timestamps (nanosecs)
  596.  *             On return contains true maximum error of timestamp
  597.  * @vblank_time: Pointer to struct timeval which should receive the timestamp
  598.  * @flags: Flags to pass to driver:
  599.  *         0 = Default,
  600.  *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl IRQ handler
  601.  * @mode: mode which defines the scanout timings
  602.  *
  603.  * Implements calculation of exact vblank timestamps from given drm_display_mode
  604.  * timings and current video scanout position of a CRTC. This can be called from
  605.  * within get_vblank_timestamp() implementation of a kms driver to implement the
  606.  * actual timestamping.
  607.  *
  608.  * Should return timestamps conforming to the OML_sync_control OpenML
  609.  * extension specification. The timestamp corresponds to the end of
  610.  * the vblank interval, aka start of scanout of topmost-leftmost display
  611.  * pixel in the following video frame.
  612.  *
  613.  * Requires support for optional dev->driver->get_scanout_position()
  614.  * in kms driver, plus a bit of setup code to provide a drm_display_mode
  615.  * that corresponds to the true scanout timing.
  616.  *
  617.  * The current implementation only handles standard video modes. It
  618.  * returns as no operation if a doublescan or interlaced video mode is
  619.  * active. Higher level code is expected to handle this.
  620.  *
  621.  * Returns:
  622.  * Negative value on error, failure or if not supported in current
  623.  * video mode:
  624.  *
  625.  * -EINVAL   - Invalid CRTC.
  626.  * -EAGAIN   - Temporary unavailable, e.g., called before initial modeset.
  627.  * -ENOTSUPP - Function not supported in current display mode.
  628.  * -EIO      - Failed, e.g., due to failed scanout position query.
  629.  *
  630.  * Returns or'ed positive status flags on success:
  631.  *
  632.  * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
  633.  * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
  634.  *
  635.  */
  636. int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
  637.                                           unsigned int pipe,
  638.                                           int *max_error,
  639.                                           struct timeval *vblank_time,
  640.                                           unsigned flags,
  641.                                           const struct drm_display_mode *mode)
  642. {
  643.         struct timeval tv_etime;
  644.         ktime_t stime, etime;
  645.         unsigned int vbl_status;
  646.         int ret = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
  647.         int vpos, hpos, i;
  648.         int delta_ns, duration_ns;
  649.  
  650.         if (pipe >= dev->num_crtcs) {
  651.                 DRM_ERROR("Invalid crtc %u\n", pipe);
  652.                 return -EINVAL;
  653.         }
  654.  
  655.         /* Scanout position query not supported? Should not happen. */
  656.         if (!dev->driver->get_scanout_position) {
  657.                 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
  658.                 return -EIO;
  659.         }
  660.  
  661.         /* If mode timing undefined, just return as no-op:
  662.          * Happens during initial modesetting of a crtc.
  663.          */
  664.         if (mode->crtc_clock == 0) {
  665.                 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
  666.                 return -EAGAIN;
  667.         }
  668.  
  669.         /* Get current scanout position with system timestamp.
  670.          * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
  671.          * if single query takes longer than max_error nanoseconds.
  672.          *
  673.          * This guarantees a tight bound on maximum error if
  674.          * code gets preempted or delayed for some reason.
  675.          */
  676.         for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
  677.                 /*
  678.                  * Get vertical and horizontal scanout position vpos, hpos,
  679.                  * and bounding timestamps stime, etime, pre/post query.
  680.                  */
  681.                 vbl_status = dev->driver->get_scanout_position(dev, pipe, flags,
  682.                                                                &vpos, &hpos,
  683.                                                                &stime, &etime,
  684.                                                                mode);
  685.  
  686.                 /* Return as no-op if scanout query unsupported or failed. */
  687.                 if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
  688.                         DRM_DEBUG("crtc %u : scanoutpos query failed [0x%x].\n",
  689.                                   pipe, vbl_status);
  690.                         return -EIO;
  691.                 }
  692.  
  693.                 /* Compute uncertainty in timestamp of scanout position query. */
  694.                 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
  695.  
  696.                 /* Accept result with <  max_error nsecs timing uncertainty. */
  697.                 if (duration_ns <= *max_error)
  698.                         break;
  699.         }
  700.  
  701.         /* Noisy system timing? */
  702.         if (i == DRM_TIMESTAMP_MAXRETRIES) {
  703.                 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
  704.                           pipe, duration_ns/1000, *max_error/1000, i);
  705.         }
  706.  
  707.         /* Return upper bound of timestamp precision error. */
  708.         *max_error = duration_ns;
  709.  
  710.         /* Check if in vblank area:
  711.          * vpos is >=0 in video scanout area, but negative
  712.          * within vblank area, counting down the number of lines until
  713.          * start of scanout.
  714.          */
  715.         if (vbl_status & DRM_SCANOUTPOS_IN_VBLANK)
  716.                 ret |= DRM_VBLANKTIME_IN_VBLANK;
  717.  
  718.         /* Convert scanout position into elapsed time at raw_time query
  719.          * since start of scanout at first display scanline. delta_ns
  720.          * can be negative if start of scanout hasn't happened yet.
  721.          */
  722.         delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
  723.                            mode->crtc_clock);
  724.  
  725.         if (!drm_timestamp_monotonic)
  726.                 etime = ktime_mono_to_real(etime);
  727.  
  728.         /* save this only for debugging purposes */
  729.         tv_etime = ktime_to_timeval(etime);
  730.         /* Subtract time delta from raw timestamp to get final
  731.          * vblank_time timestamp for end of vblank.
  732.          */
  733.         if (delta_ns < 0)
  734.                 etime = ktime_add_ns(etime, -delta_ns);
  735.         else
  736.                 etime = ktime_sub_ns(etime, delta_ns);
  737.         *vblank_time = ktime_to_timeval(etime);
  738.  
  739.         DRM_DEBUG_VBL("crtc %u : v 0x%x p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
  740.                       pipe, vbl_status, hpos, vpos,
  741.                       (long)tv_etime.tv_sec, (long)tv_etime.tv_usec,
  742.                       (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
  743.                       duration_ns/1000, i);
  744.  
  745.         return ret;
  746. }
  747. EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
  748.  
  749. static struct timeval get_drm_timestamp(void)
  750. {
  751.         ktime_t now;
  752.  
  753.         now = drm_timestamp_monotonic ? ktime_get() : ktime_get_real();
  754.         return ktime_to_timeval(now);
  755. }
  756.  
  757. /**
  758.  * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
  759.  *                             vblank interval
  760.  * @dev: DRM device
  761.  * @pipe: index of CRTC whose vblank timestamp to retrieve
  762.  * @tvblank: Pointer to target struct timeval which should receive the timestamp
  763.  * @flags: Flags to pass to driver:
  764.  *         0 = Default,
  765.  *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl IRQ handler
  766.  *
  767.  * Fetches the system timestamp corresponding to the time of the most recent
  768.  * vblank interval on specified CRTC. May call into kms-driver to
  769.  * compute the timestamp with a high-precision GPU specific method.
  770.  *
  771.  * Returns zero if timestamp originates from uncorrected do_gettimeofday()
  772.  * call, i.e., it isn't very precisely locked to the true vblank.
  773.  *
  774.  * Returns:
  775.  * True if timestamp is considered to be very precise, false otherwise.
  776.  */
  777. static bool
  778. drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  779.                           struct timeval *tvblank, unsigned flags)
  780. {
  781.         int ret;
  782.  
  783.         /* Define requested maximum error on timestamps (nanoseconds). */
  784.         int max_error = (int) drm_timestamp_precision * 1000;
  785.  
  786.         /* Query driver if possible and precision timestamping enabled. */
  787.         if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
  788.                 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
  789.                                                         tvblank, flags);
  790.                 if (ret > 0)
  791.                         return true;
  792.         }
  793.  
  794.         /* GPU high precision timestamp query unsupported or failed.
  795.          * Return current monotonic/gettimeofday timestamp as best estimate.
  796.          */
  797.         *tvblank = get_drm_timestamp();
  798.  
  799.         return false;
  800. }
  801.  
  802. /**
  803.  * drm_vblank_count - retrieve "cooked" vblank counter value
  804.  * @dev: DRM device
  805.  * @pipe: index of CRTC for which to retrieve the counter
  806.  *
  807.  * Fetches the "cooked" vblank count value that represents the number of
  808.  * vblank events since the system was booted, including lost events due to
  809.  * modesetting activity.
  810.  *
  811.  * This is the legacy version of drm_crtc_vblank_count().
  812.  *
  813.  * Returns:
  814.  * The software vblank counter.
  815.  */
  816. u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
  817. {
  818.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  819.  
  820.         if (WARN_ON(pipe >= dev->num_crtcs))
  821.                 return 0;
  822.  
  823.         return vblank->count;
  824. }
  825. EXPORT_SYMBOL(drm_vblank_count);
  826.  
  827. /**
  828.  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
  829.  * @crtc: which counter to retrieve
  830.  *
  831.  * Fetches the "cooked" vblank count value that represents the number of
  832.  * vblank events since the system was booted, including lost events due to
  833.  * modesetting activity.
  834.  *
  835.  * This is the native KMS version of drm_vblank_count().
  836.  *
  837.  * Returns:
  838.  * The software vblank counter.
  839.  */
  840. u32 drm_crtc_vblank_count(struct drm_crtc *crtc)
  841. {
  842.         return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
  843. }
  844. EXPORT_SYMBOL(drm_crtc_vblank_count);
  845.  
  846. /**
  847.  * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
  848.  *     system timestamp corresponding to that vblank counter value.
  849.  * @dev: DRM device
  850.  * @pipe: index of CRTC whose counter to retrieve
  851.  * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
  852.  *
  853.  * Fetches the "cooked" vblank count value that represents the number of
  854.  * vblank events since the system was booted, including lost events due to
  855.  * modesetting activity. Returns corresponding system timestamp of the time
  856.  * of the vblank interval that corresponds to the current vblank counter value.
  857.  *
  858.  * This is the legacy version of drm_crtc_vblank_count_and_time().
  859.  */
  860. u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
  861.                               struct timeval *vblanktime)
  862. {
  863.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  864.         int count = DRM_TIMESTAMP_MAXRETRIES;
  865.         u32 cur_vblank;
  866.  
  867.         if (WARN_ON(pipe >= dev->num_crtcs))
  868.                 return 0;
  869.  
  870.         /*
  871.          * Vblank timestamps are read lockless. To ensure consistency the vblank
  872.          * counter is rechecked and ordering is ensured using memory barriers.
  873.          * This works like a seqlock. The write-side barriers are in store_vblank.
  874.          */
  875.         do {
  876.                 cur_vblank = vblank->count;
  877.                 smp_rmb();
  878.                 *vblanktime = vblanktimestamp(dev, pipe, cur_vblank);
  879.                 smp_rmb();
  880.         } while (cur_vblank != vblank->count && --count > 0);
  881.  
  882.         return cur_vblank;
  883. }
  884. EXPORT_SYMBOL(drm_vblank_count_and_time);
  885.  
  886. /**
  887.  * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
  888.  *     and the system timestamp corresponding to that vblank counter value
  889.  * @crtc: which counter to retrieve
  890.  * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
  891.  *
  892.  * Fetches the "cooked" vblank count value that represents the number of
  893.  * vblank events since the system was booted, including lost events due to
  894.  * modesetting activity. Returns corresponding system timestamp of the time
  895.  * of the vblank interval that corresponds to the current vblank counter value.
  896.  *
  897.  * This is the native KMS version of drm_vblank_count_and_time().
  898.  */
  899. u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
  900.                                    struct timeval *vblanktime)
  901. {
  902.         return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
  903.                                          vblanktime);
  904. }
  905. EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
  906.  
  907. static void send_vblank_event(struct drm_device *dev,
  908.                 struct drm_pending_vblank_event *e,
  909.                 unsigned long seq, struct timeval *now)
  910. {
  911.         e->event.sequence = seq;
  912.         e->event.tv_sec = now->tv_sec;
  913.         e->event.tv_usec = now->tv_usec;
  914.  
  915.  
  916. }
  917.  
  918. /**
  919.  * drm_arm_vblank_event - arm vblank event after pageflip
  920.  * @dev: DRM device
  921.  * @pipe: CRTC index
  922.  * @e: the event to prepare to send
  923.  *
  924.  * A lot of drivers need to generate vblank events for the very next vblank
  925.  * interrupt. For example when the page flip interrupt happens when the page
  926.  * flip gets armed, but not when it actually executes within the next vblank
  927.  * period. This helper function implements exactly the required vblank arming
  928.  * behaviour.
  929.  *
  930.  * Caller must hold event lock. Caller must also hold a vblank reference for
  931.  * the event @e, which will be dropped when the next vblank arrives.
  932.  *
  933.  * This is the legacy version of drm_crtc_arm_vblank_event().
  934.  */
  935. void drm_arm_vblank_event(struct drm_device *dev, unsigned int pipe,
  936.                           struct drm_pending_vblank_event *e)
  937. {
  938.         assert_spin_locked(&dev->event_lock);
  939.  
  940.         e->pipe = pipe;
  941.         e->event.sequence = drm_vblank_count(dev, pipe);
  942.         list_add_tail(&e->base.link, &dev->vblank_event_list);
  943. }
  944. EXPORT_SYMBOL(drm_arm_vblank_event);
  945.  
  946. /**
  947.  * drm_crtc_arm_vblank_event - arm vblank event after pageflip
  948.  * @crtc: the source CRTC of the vblank event
  949.  * @e: the event to send
  950.  *
  951.  * A lot of drivers need to generate vblank events for the very next vblank
  952.  * interrupt. For example when the page flip interrupt happens when the page
  953.  * flip gets armed, but not when it actually executes within the next vblank
  954.  * period. This helper function implements exactly the required vblank arming
  955.  * behaviour.
  956.  *
  957.  * Caller must hold event lock. Caller must also hold a vblank reference for
  958.  * the event @e, which will be dropped when the next vblank arrives.
  959.  *
  960.  * This is the native KMS version of drm_arm_vblank_event().
  961.  */
  962. void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
  963.                                struct drm_pending_vblank_event *e)
  964. {
  965.         drm_arm_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
  966. }
  967. EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
  968.  
  969. /**
  970.  * drm_send_vblank_event - helper to send vblank event after pageflip
  971.  * @dev: DRM device
  972.  * @pipe: CRTC index
  973.  * @e: the event to send
  974.  *
  975.  * Updates sequence # and timestamp on event, and sends it to userspace.
  976.  * Caller must hold event lock.
  977.  *
  978.  * This is the legacy version of drm_crtc_send_vblank_event().
  979.  */
  980. void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe,
  981.                            struct drm_pending_vblank_event *e)
  982. {
  983.         struct timeval now;
  984.         unsigned int seq;
  985.  
  986.         if (dev->num_crtcs > 0) {
  987.                 seq = drm_vblank_count_and_time(dev, pipe, &now);
  988.         } else {
  989.                 seq = 0;
  990.  
  991.                 now = get_drm_timestamp();
  992.         }
  993.         e->pipe = pipe;
  994.         send_vblank_event(dev, e, seq, &now);
  995. }
  996. EXPORT_SYMBOL(drm_send_vblank_event);
  997.  
  998. /**
  999.  * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
  1000.  * @crtc: the source CRTC of the vblank event
  1001.  * @e: the event to send
  1002.  *
  1003.  * Updates sequence # and timestamp on event, and sends it to userspace.
  1004.  * Caller must hold event lock.
  1005.  *
  1006.  * This is the native KMS version of drm_send_vblank_event().
  1007.  */
  1008. void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
  1009.                                 struct drm_pending_vblank_event *e)
  1010. {
  1011.         drm_send_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
  1012. }
  1013. EXPORT_SYMBOL(drm_crtc_send_vblank_event);
  1014.  
  1015. /**
  1016.  * drm_vblank_enable - enable the vblank interrupt on a CRTC
  1017.  * @dev: DRM device
  1018.  * @pipe: CRTC index
  1019.  *
  1020.  * Returns:
  1021.  * Zero on success or a negative error code on failure.
  1022.  */
  1023. static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
  1024. {
  1025.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1026.         int ret = 0;
  1027.  
  1028.         assert_spin_locked(&dev->vbl_lock);
  1029.  
  1030.         spin_lock(&dev->vblank_time_lock);
  1031.  
  1032.         if (!vblank->enabled) {
  1033.                 /*
  1034.                  * Enable vblank irqs under vblank_time_lock protection.
  1035.                  * All vblank count & timestamp updates are held off
  1036.                  * until we are done reinitializing master counter and
  1037.                  * timestamps. Filtercode in drm_handle_vblank() will
  1038.                  * prevent double-accounting of same vblank interval.
  1039.                  */
  1040.                 ret = dev->driver->enable_vblank(dev, pipe);
  1041.                 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
  1042.                 if (ret)
  1043.                         atomic_dec(&vblank->refcount);
  1044.                 else {
  1045.                         vblank->enabled = true;
  1046.                         drm_update_vblank_count(dev, pipe, 0);
  1047.                 }
  1048.         }
  1049.  
  1050.         spin_unlock(&dev->vblank_time_lock);
  1051.  
  1052.         return ret;
  1053. }
  1054.  
  1055. /**
  1056.  * drm_vblank_get - get a reference count on vblank events
  1057.  * @dev: DRM device
  1058.  * @pipe: index of CRTC to own
  1059.  *
  1060.  * Acquire a reference count on vblank events to avoid having them disabled
  1061.  * while in use.
  1062.  *
  1063.  * This is the legacy version of drm_crtc_vblank_get().
  1064.  *
  1065.  * Returns:
  1066.  * Zero on success or a negative error code on failure.
  1067.  */
  1068. int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
  1069. {
  1070.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1071.         unsigned long irqflags;
  1072.         int ret = 0;
  1073.  
  1074.         if (!dev->num_crtcs)
  1075.                 return -EINVAL;
  1076.  
  1077.         if (WARN_ON(pipe >= dev->num_crtcs))
  1078.                 return -EINVAL;
  1079.  
  1080.         spin_lock_irqsave(&dev->vbl_lock, irqflags);
  1081.         /* Going from 0->1 means we have to enable interrupts again */
  1082.         if (atomic_add_return(1, &vblank->refcount) == 1) {
  1083.                 ret = drm_vblank_enable(dev, pipe);
  1084.         } else {
  1085.                 if (!vblank->enabled) {
  1086.                         atomic_dec(&vblank->refcount);
  1087.                         ret = -EINVAL;
  1088.                 }
  1089.         }
  1090.         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  1091.  
  1092.         return ret;
  1093. }
  1094. EXPORT_SYMBOL(drm_vblank_get);
  1095.  
  1096. /**
  1097.  * drm_crtc_vblank_get - get a reference count on vblank events
  1098.  * @crtc: which CRTC to own
  1099.  *
  1100.  * Acquire a reference count on vblank events to avoid having them disabled
  1101.  * while in use.
  1102.  *
  1103.  * This is the native kms version of drm_vblank_get().
  1104.  *
  1105.  * Returns:
  1106.  * Zero on success or a negative error code on failure.
  1107.  */
  1108. int drm_crtc_vblank_get(struct drm_crtc *crtc)
  1109. {
  1110.         return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
  1111. }
  1112. EXPORT_SYMBOL(drm_crtc_vblank_get);
  1113.  
  1114. /**
  1115.  * drm_vblank_put - release ownership of vblank events
  1116.  * @dev: DRM device
  1117.  * @pipe: index of CRTC to release
  1118.  *
  1119.  * Release ownership of a given vblank counter, turning off interrupts
  1120.  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
  1121.  *
  1122.  * This is the legacy version of drm_crtc_vblank_put().
  1123.  */
  1124. void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
  1125. {
  1126.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1127.  
  1128.         if (WARN_ON(pipe >= dev->num_crtcs))
  1129.                 return;
  1130.  
  1131.         if (WARN_ON(atomic_read(&vblank->refcount) == 0))
  1132.                 return;
  1133.  
  1134.         /* Last user schedules interrupt disable */
  1135.         if (atomic_dec_and_test(&vblank->refcount)) {
  1136.                 if (drm_vblank_offdelay == 0)
  1137.                         return;
  1138.                 else
  1139.                         vblank_disable_fn((unsigned long)vblank);
  1140.         }
  1141. }
  1142. EXPORT_SYMBOL(drm_vblank_put);
  1143.  
  1144. /**
  1145.  * drm_crtc_vblank_put - give up ownership of vblank events
  1146.  * @crtc: which counter to give up
  1147.  *
  1148.  * Release ownership of a given vblank counter, turning off interrupts
  1149.  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
  1150.  *
  1151.  * This is the native kms version of drm_vblank_put().
  1152.  */
  1153. void drm_crtc_vblank_put(struct drm_crtc *crtc)
  1154. {
  1155.         drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
  1156. }
  1157. EXPORT_SYMBOL(drm_crtc_vblank_put);
  1158.  
  1159. /**
  1160.  * drm_wait_one_vblank - wait for one vblank
  1161.  * @dev: DRM device
  1162.  * @pipe: CRTC index
  1163.  *
  1164.  * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
  1165.  * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
  1166.  * due to lack of driver support or because the crtc is off.
  1167.  */
  1168. void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
  1169. {
  1170.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1171.         int ret;
  1172.         u32 last;
  1173.  
  1174.         if (WARN_ON(pipe >= dev->num_crtcs))
  1175.                 return;
  1176.  
  1177.         ret = drm_vblank_get(dev, pipe);
  1178.         if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
  1179.                 return;
  1180.  
  1181.         last = drm_vblank_count(dev, pipe);
  1182.  
  1183.         ret = wait_event_timeout(vblank->queue,
  1184.                                  last != drm_vblank_count(dev, pipe),
  1185.                                  msecs_to_jiffies(100));
  1186.  
  1187.         WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
  1188.  
  1189.         drm_vblank_put(dev, pipe);
  1190. }
  1191. EXPORT_SYMBOL(drm_wait_one_vblank);
  1192.  
  1193. /**
  1194.  * drm_crtc_wait_one_vblank - wait for one vblank
  1195.  * @crtc: DRM crtc
  1196.  *
  1197.  * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
  1198.  * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
  1199.  * due to lack of driver support or because the crtc is off.
  1200.  */
  1201. void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
  1202. {
  1203.         drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
  1204. }
  1205. EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  1206.  
  1207. /**
  1208.  * drm_vblank_off - disable vblank events on a CRTC
  1209.  * @dev: DRM device
  1210.  * @pipe: CRTC index
  1211.  *
  1212.  * Drivers can use this function to shut down the vblank interrupt handling when
  1213.  * disabling a crtc. This function ensures that the latest vblank frame count is
  1214.  * stored so that drm_vblank_on() can restore it again.
  1215.  *
  1216.  * Drivers must use this function when the hardware vblank counter can get
  1217.  * reset, e.g. when suspending.
  1218.  *
  1219.  * This is the legacy version of drm_crtc_vblank_off().
  1220.  */
  1221. void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
  1222. {
  1223.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1224.         struct drm_pending_vblank_event *e, *t;
  1225.         struct timeval now;
  1226.         unsigned long irqflags;
  1227.         unsigned int seq;
  1228.  
  1229.         if (WARN_ON(pipe >= dev->num_crtcs))
  1230.                 return;
  1231.  
  1232.         spin_lock_irqsave(&dev->event_lock, irqflags);
  1233.  
  1234.         spin_lock(&dev->vbl_lock);
  1235.         DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
  1236.                       pipe, vblank->enabled, vblank->inmodeset);
  1237.  
  1238.         /* Avoid redundant vblank disables without previous drm_vblank_on(). */
  1239.         if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
  1240.                 vblank_disable_and_save(dev, pipe);
  1241.  
  1242.         wake_up(&vblank->queue);
  1243.  
  1244.         /*
  1245.          * Prevent subsequent drm_vblank_get() from re-enabling
  1246.          * the vblank interrupt by bumping the refcount.
  1247.          */
  1248.         if (!vblank->inmodeset) {
  1249.                 atomic_inc(&vblank->refcount);
  1250.                 vblank->inmodeset = 1;
  1251.         }
  1252.         spin_unlock(&dev->vbl_lock);
  1253.  
  1254.         /* Send any queued vblank events, lest the natives grow disquiet */
  1255.         seq = drm_vblank_count_and_time(dev, pipe, &now);
  1256.  
  1257.         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
  1258.                 if (e->pipe != pipe)
  1259.                         continue;
  1260.                 DRM_DEBUG("Sending premature vblank event on disable: "
  1261.                           "wanted %d, current %d\n",
  1262.                           e->event.sequence, seq);
  1263.                 list_del(&e->base.link);
  1264.                 drm_vblank_put(dev, pipe);
  1265.                 send_vblank_event(dev, e, seq, &now);
  1266.         }
  1267.         spin_unlock_irqrestore(&dev->event_lock, irqflags);
  1268. }
  1269. EXPORT_SYMBOL(drm_vblank_off);
  1270.  
  1271. /**
  1272.  * drm_crtc_vblank_off - disable vblank events on a CRTC
  1273.  * @crtc: CRTC in question
  1274.  *
  1275.  * Drivers can use this function to shut down the vblank interrupt handling when
  1276.  * disabling a crtc. This function ensures that the latest vblank frame count is
  1277.  * stored so that drm_vblank_on can restore it again.
  1278.  *
  1279.  * Drivers must use this function when the hardware vblank counter can get
  1280.  * reset, e.g. when suspending.
  1281.  *
  1282.  * This is the native kms version of drm_vblank_off().
  1283.  */
  1284. void drm_crtc_vblank_off(struct drm_crtc *crtc)
  1285. {
  1286.         drm_vblank_off(crtc->dev, drm_crtc_index(crtc));
  1287. }
  1288. EXPORT_SYMBOL(drm_crtc_vblank_off);
  1289.  
  1290. /**
  1291.  * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
  1292.  * @crtc: CRTC in question
  1293.  *
  1294.  * Drivers can use this function to reset the vblank state to off at load time.
  1295.  * Drivers should use this together with the drm_crtc_vblank_off() and
  1296.  * drm_crtc_vblank_on() functions. The difference compared to
  1297.  * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
  1298.  * and hence doesn't need to call any driver hooks.
  1299.  */
  1300. void drm_crtc_vblank_reset(struct drm_crtc *crtc)
  1301. {
  1302.         struct drm_device *dev = crtc->dev;
  1303.         unsigned long irqflags;
  1304.         unsigned int pipe = drm_crtc_index(crtc);
  1305.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1306.  
  1307.         spin_lock_irqsave(&dev->vbl_lock, irqflags);
  1308.         /*
  1309.          * Prevent subsequent drm_vblank_get() from enabling the vblank
  1310.          * interrupt by bumping the refcount.
  1311.          */
  1312.         if (!vblank->inmodeset) {
  1313.                 atomic_inc(&vblank->refcount);
  1314.                 vblank->inmodeset = 1;
  1315.         }
  1316.         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  1317.  
  1318.         WARN_ON(!list_empty(&dev->vblank_event_list));
  1319. }
  1320. EXPORT_SYMBOL(drm_crtc_vblank_reset);
  1321.  
  1322. /**
  1323.  * drm_vblank_on - enable vblank events on a CRTC
  1324.  * @dev: DRM device
  1325.  * @pipe: CRTC index
  1326.  *
  1327.  * This functions restores the vblank interrupt state captured with
  1328.  * drm_vblank_off() again. Note that calls to drm_vblank_on() and
  1329.  * drm_vblank_off() can be unbalanced and so can also be unconditionally called
  1330.  * in driver load code to reflect the current hardware state of the crtc.
  1331.  *
  1332.  * This is the legacy version of drm_crtc_vblank_on().
  1333.  */
  1334. void drm_vblank_on(struct drm_device *dev, unsigned int pipe)
  1335. {
  1336.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1337.         unsigned long irqflags;
  1338.  
  1339.         if (WARN_ON(pipe >= dev->num_crtcs))
  1340.                 return;
  1341.  
  1342.         spin_lock_irqsave(&dev->vbl_lock, irqflags);
  1343.         DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
  1344.                       pipe, vblank->enabled, vblank->inmodeset);
  1345.  
  1346.         /* Drop our private "prevent drm_vblank_get" refcount */
  1347.         if (vblank->inmodeset) {
  1348.                 atomic_dec(&vblank->refcount);
  1349.                 vblank->inmodeset = 0;
  1350.         }
  1351.  
  1352.         drm_reset_vblank_timestamp(dev, pipe);
  1353.  
  1354.         /*
  1355.          * re-enable interrupts if there are users left, or the
  1356.          * user wishes vblank interrupts to be enabled all the time.
  1357.          */
  1358.         if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
  1359.                 WARN_ON(drm_vblank_enable(dev, pipe));
  1360.         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  1361. }
  1362. EXPORT_SYMBOL(drm_vblank_on);
  1363.  
  1364. /**
  1365.  * drm_crtc_vblank_on - enable vblank events on a CRTC
  1366.  * @crtc: CRTC in question
  1367.  *
  1368.  * This functions restores the vblank interrupt state captured with
  1369.  * drm_vblank_off() again. Note that calls to drm_vblank_on() and
  1370.  * drm_vblank_off() can be unbalanced and so can also be unconditionally called
  1371.  * in driver load code to reflect the current hardware state of the crtc.
  1372.  *
  1373.  * This is the native kms version of drm_vblank_on().
  1374.  */
  1375. void drm_crtc_vblank_on(struct drm_crtc *crtc)
  1376. {
  1377.         drm_vblank_on(crtc->dev, drm_crtc_index(crtc));
  1378. }
  1379. EXPORT_SYMBOL(drm_crtc_vblank_on);
  1380.  
  1381. /**
  1382.  * drm_vblank_pre_modeset - account for vblanks across mode sets
  1383.  * @dev: DRM device
  1384.  * @pipe: CRTC index
  1385.  *
  1386.  * Account for vblank events across mode setting events, which will likely
  1387.  * reset the hardware frame counter.
  1388.  *
  1389.  * This is done by grabbing a temporary vblank reference to ensure that the
  1390.  * vblank interrupt keeps running across the modeset sequence. With this the
  1391.  * software-side vblank frame counting will ensure that there are no jumps or
  1392.  * discontinuities.
  1393.  *
  1394.  * Unfortunately this approach is racy and also doesn't work when the vblank
  1395.  * interrupt stops running, e.g. across system suspend resume. It is therefore
  1396.  * highly recommended that drivers use the newer drm_vblank_off() and
  1397.  * drm_vblank_on() instead. drm_vblank_pre_modeset() only works correctly when
  1398.  * using "cooked" software vblank frame counters and not relying on any hardware
  1399.  * counters.
  1400.  *
  1401.  * Drivers must call drm_vblank_post_modeset() when re-enabling the same crtc
  1402.  * again.
  1403.  */
  1404. void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe)
  1405. {
  1406.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1407.  
  1408.         /* vblank is not initialized (IRQ not installed ?), or has been freed */
  1409.         if (!dev->num_crtcs)
  1410.                 return;
  1411.  
  1412.         if (WARN_ON(pipe >= dev->num_crtcs))
  1413.                 return;
  1414.  
  1415.         /*
  1416.          * To avoid all the problems that might happen if interrupts
  1417.          * were enabled/disabled around or between these calls, we just
  1418.          * have the kernel take a reference on the CRTC (just once though
  1419.          * to avoid corrupting the count if multiple, mismatch calls occur),
  1420.          * so that interrupts remain enabled in the interim.
  1421.          */
  1422.         if (!vblank->inmodeset) {
  1423.                 vblank->inmodeset = 0x1;
  1424.                 if (drm_vblank_get(dev, pipe) == 0)
  1425.                         vblank->inmodeset |= 0x2;
  1426.         }
  1427. }
  1428. EXPORT_SYMBOL(drm_vblank_pre_modeset);
  1429.  
  1430. /**
  1431.  * drm_vblank_post_modeset - undo drm_vblank_pre_modeset changes
  1432.  * @dev: DRM device
  1433.  * @pipe: CRTC index
  1434.  *
  1435.  * This function again drops the temporary vblank reference acquired in
  1436.  * drm_vblank_pre_modeset.
  1437.  */
  1438. void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe)
  1439. {
  1440.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1441.         unsigned long irqflags;
  1442.  
  1443.         /* vblank is not initialized (IRQ not installed ?), or has been freed */
  1444.         if (!dev->num_crtcs)
  1445.                 return;
  1446.  
  1447.         if (WARN_ON(pipe >= dev->num_crtcs))
  1448.                 return;
  1449.  
  1450.         if (vblank->inmodeset) {
  1451.                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
  1452.                 dev->vblank_disable_allowed = true;
  1453.                 drm_reset_vblank_timestamp(dev, pipe);
  1454.                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  1455.  
  1456.                 if (vblank->inmodeset & 0x2)
  1457.                         drm_vblank_put(dev, pipe);
  1458.  
  1459.                 vblank->inmodeset = 0;
  1460.         }
  1461. }
  1462. EXPORT_SYMBOL(drm_vblank_post_modeset);
  1463.  
  1464. static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
  1465. {
  1466.         struct drm_pending_vblank_event *e, *t;
  1467.         struct timeval now;
  1468.         unsigned int seq;
  1469.  
  1470.         assert_spin_locked(&dev->event_lock);
  1471.  
  1472.         seq = drm_vblank_count_and_time(dev, pipe, &now);
  1473.  
  1474.         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
  1475.                 if (e->pipe != pipe)
  1476.                         continue;
  1477.                 if ((seq - e->event.sequence) > (1<<23))
  1478.                         continue;
  1479.  
  1480.                 DRM_DEBUG("vblank event on %d, current %d\n",
  1481.                           e->event.sequence, seq);
  1482.  
  1483.                 list_del(&e->base.link);
  1484.                 drm_vblank_put(dev, pipe);
  1485. //       send_vblank_event(dev, e, seq, &now);
  1486.         }
  1487.  
  1488. }
  1489.  
  1490. /**
  1491.  * drm_handle_vblank - handle a vblank event
  1492.  * @dev: DRM device
  1493.  * @pipe: index of CRTC where this event occurred
  1494.  *
  1495.  * Drivers should call this routine in their vblank interrupt handlers to
  1496.  * update the vblank counter and send any signals that may be pending.
  1497.  *
  1498.  * This is the legacy version of drm_crtc_handle_vblank().
  1499.  */
  1500. bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
  1501. {
  1502.         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1503.         unsigned long irqflags;
  1504.  
  1505.         if (WARN_ON_ONCE(!dev->num_crtcs))
  1506.                 return false;
  1507.  
  1508.         if (WARN_ON(pipe >= dev->num_crtcs))
  1509.                 return false;
  1510.  
  1511.         spin_lock_irqsave(&dev->event_lock, irqflags);
  1512.  
  1513.         /* Need timestamp lock to prevent concurrent execution with
  1514.          * vblank enable/disable, as this would cause inconsistent
  1515.          * or corrupted timestamps and vblank counts.
  1516.          */
  1517.         spin_lock(&dev->vblank_time_lock);
  1518.  
  1519.         /* Vblank irq handling disabled. Nothing to do. */
  1520.         if (!vblank->enabled) {
  1521.                 spin_unlock(&dev->vblank_time_lock);
  1522.                 spin_unlock_irqrestore(&dev->event_lock, irqflags);
  1523.                 return false;
  1524.         }
  1525.  
  1526.         drm_update_vblank_count(dev, pipe, DRM_CALLED_FROM_VBLIRQ);
  1527.  
  1528.         spin_unlock(&dev->vblank_time_lock);
  1529.  
  1530.         wake_up(&vblank->queue);
  1531.         drm_handle_vblank_events(dev, pipe);
  1532.  
  1533.         spin_unlock_irqrestore(&dev->event_lock, irqflags);
  1534.  
  1535.         return true;
  1536. }
  1537. EXPORT_SYMBOL(drm_handle_vblank);
  1538.  
  1539. /**
  1540.  * drm_crtc_handle_vblank - handle a vblank event
  1541.  * @crtc: where this event occurred
  1542.  *
  1543.  * Drivers should call this routine in their vblank interrupt handlers to
  1544.  * update the vblank counter and send any signals that may be pending.
  1545.  *
  1546.  * This is the native KMS version of drm_handle_vblank().
  1547.  *
  1548.  * Returns:
  1549.  * True if the event was successfully handled, false on failure.
  1550.  */
  1551. bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
  1552. {
  1553.         return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
  1554. }
  1555. EXPORT_SYMBOL(drm_crtc_handle_vblank);
  1556.  
  1557. /**
  1558.  * drm_vblank_no_hw_counter - "No hw counter" implementation of .get_vblank_counter()
  1559.  * @dev: DRM device
  1560.  * @pipe: CRTC for which to read the counter
  1561.  *
  1562.  * Drivers can plug this into the .get_vblank_counter() function if
  1563.  * there is no useable hardware frame counter available.
  1564.  *
  1565.  * Returns:
  1566.  * 0
  1567.  */
  1568. u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
  1569. {
  1570.         return 0;
  1571. }
  1572. EXPORT_SYMBOL(drm_vblank_no_hw_counter);
  1573.