Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 3d29b842e58fbca2c13a9f458fddbaa535c6e578

Authored by Eugeni Dodonov 2012-01-18 00:43:53 +0800

Committed by Daniel Vetter 2012-01-18 03:01:45 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

drm/i915: add a LLC feature flag in device description

LLC is not SNB/IVB-specific, so we should check for it in a more generic
way.

Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Eric Anholt <eric@anholt.net>
Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>
Signed-off-by: Eugeni Dodonov <eugeni.dodonov@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

Showing 6 changed files with 13 additions and 2 deletions Inline Diff

drivers/gpu/drm/i915/i915_debugfs.c
drivers/gpu/drm/i915/i915_dma.c
drivers/gpu/drm/i915/i915_drv.c
drivers/gpu/drm/i915/i915_drv.h
drivers/gpu/drm/i915/i915_gem.c
include/drm/i915_drm.h

drivers/gpu/drm/i915/i915_debugfs.c

Diff comments View file @ 3d29b84

 /*
  * Copyright © 2008 Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  *
  * Authors:
  *    Eric Anholt <eric@anholt.net>
  *    Keith Packard <keithp@keithp.com>
  *
  */
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
 #include <linux/slab.h>
 #include <linux/export.h>
 #include "drmP.h"
 #include "drm.h"
 #include "intel_drv.h"
 #include "intel_ringbuffer.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 #define DRM_I915_RING_DEBUG 1
 #if defined(CONFIG_DEBUG_FS)
 enum {
 	ACTIVE_LIST,
 	FLUSHING_LIST,
 	INACTIVE_LIST,
 	PINNED_LIST,
 	DEFERRED_FREE_LIST,
 };
 static const char *yesno(int v)
 {
 	return v ? "yes" : "no";
 }
 static int i915_capabilities(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	const struct intel_device_info *info = INTEL_INFO(dev);
 	seq_printf(m, "gen: %d\n", info->gen);
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev));
 #define B(x) seq_printf(m, #x ": %s\n", yesno(info->x))
 	B(is_mobile);
 	B(is_i85x);
 	B(is_i915g);
 	B(is_i945gm);
 	B(is_g33);
 	B(need_gfx_hws);
 	B(is_g4x);
 	B(is_pineview);
 	B(is_broadwater);
 	B(is_crestline);
 	B(has_fbc);
 	B(has_pipe_cxsr);
 	B(has_hotplug);
 	B(cursor_needs_physical);
 	B(has_overlay);
 	B(overlay_needs_physical);
 	B(supports_tv);
 	B(has_bsd_ring);
 	B(has_blt_ring);
+	B(has_llc);
 #undef B
 	return 0;
 }
 static const char *get_pin_flag(struct drm_i915_gem_object *obj)
 {
 	if (obj->user_pin_count > 0)
 		return "P";
 	else if (obj->pin_count > 0)
 		return "p";
 	else
 		return " ";
 }
 static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
 {
 	switch (obj->tiling_mode) {
 	default:
 	case I915_TILING_NONE: return " ";
 	case I915_TILING_X: return "X";
 	case I915_TILING_Y: return "Y";
 	}
 }
 static const char *cache_level_str(int type)
 {
 	switch (type) {
 	case I915_CACHE_NONE: return " uncached";
 	case I915_CACHE_LLC: return " snooped (LLC)";
 	case I915_CACHE_LLC_MLC: return " snooped (LLC+MLC)";
 	default: return "";
 	}
 }
 static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
 	seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s%s",
 		   &obj->base,
 		   get_pin_flag(obj),
 		   get_tiling_flag(obj),
 		   obj->base.size,
 		   obj->base.read_domains,
 		   obj->base.write_domain,
 		   obj->last_rendering_seqno,
 		   obj->last_fenced_seqno,
 		   cache_level_str(obj->cache_level),
 		   obj->dirty ? " dirty" : "",
 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		seq_printf(m, " (name: %d)", obj->base.name);
 	if (obj->fence_reg != I915_FENCE_REG_NONE)
 		seq_printf(m, " (fence: %d)", obj->fence_reg);
 	if (obj->gtt_space != NULL)
 		seq_printf(m, " (gtt offset: %08x, size: %08x)",
 			   obj->gtt_offset, (unsigned int)obj->gtt_space->size);
 	if (obj->pin_mappable || obj->fault_mappable) {
 		char s[3], *t = s;
 		if (obj->pin_mappable)
 			*t++ = 'p';
 		if (obj->fault_mappable)
 			*t++ = 'f';
 		*t = '\0';
 		seq_printf(m, " (%s mappable)", s);
 	}
 	if (obj->ring != NULL)
 		seq_printf(m, " (%s)", obj->ring->name);
 }
 static int i915_gem_object_list_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	uintptr_t list = (uintptr_t) node->info_ent->data;
 	struct list_head *head;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	size_t total_obj_size, total_gtt_size;
 	int count, ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	switch (list) {
 	case ACTIVE_LIST:
 		seq_printf(m, "Active:\n");
 		head = &dev_priv->mm.active_list;
 		break;
 	case INACTIVE_LIST:
 		seq_printf(m, "Inactive:\n");
 		head = &dev_priv->mm.inactive_list;
 		break;
 	case PINNED_LIST:
 		seq_printf(m, "Pinned:\n");
 		head = &dev_priv->mm.pinned_list;
 		break;
 	case FLUSHING_LIST:
 		seq_printf(m, "Flushing:\n");
 		head = &dev_priv->mm.flushing_list;
 		break;
 	case DEFERRED_FREE_LIST:
 		seq_printf(m, "Deferred free:\n");
 		head = &dev_priv->mm.deferred_free_list;
 		break;
 	default:
 		mutex_unlock(&dev->struct_mutex);
 		return -EINVAL;
 	}
 	total_obj_size = total_gtt_size = count = 0;
 	list_for_each_entry(obj, head, mm_list) {
 		seq_printf(m, "   ");
 		describe_obj(m, obj);
 		seq_printf(m, "\n");
 		total_obj_size += obj->base.size;
 		total_gtt_size += obj->gtt_space->size;
 		count++;
 	}
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n",
 		   count, total_obj_size, total_gtt_size);
 	return 0;
 }
 #define count_objects(list, member) do { \
 	list_for_each_entry(obj, list, member) { \
 		size += obj->gtt_space->size; \
 		++count; \
 		if (obj->map_and_fenceable) { \
 			mappable_size += obj->gtt_space->size; \
 			++mappable_count; \
 		} \
 	} \
 } while (0)
 static int i915_gem_object_info(struct seq_file *m, void* data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 count, mappable_count;
 	size_t size, mappable_size;
 	struct drm_i915_gem_object *obj;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	seq_printf(m, "%u objects, %zu bytes\n",
 		   dev_priv->mm.object_count,
 		   dev_priv->mm.object_memory);
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.gtt_list, gtt_list);
 	seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
 		   count, mappable_count, size, mappable_size);
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.active_list, mm_list);
 	count_objects(&dev_priv->mm.flushing_list, mm_list);
 	seq_printf(m, "  %u [%u] active objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.pinned_list, mm_list);
 	seq_printf(m, "  %u [%u] pinned objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.inactive_list, mm_list);
 	seq_printf(m, "  %u [%u] inactive objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.deferred_free_list, mm_list);
 	seq_printf(m, "  %u [%u] freed objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 	size = count = mappable_size = mappable_count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 		if (obj->fault_mappable) {
 			size += obj->gtt_space->size;
 			++count;
 		}
 		if (obj->pin_mappable) {
 			mappable_size += obj->gtt_space->size;
 			++mappable_count;
 		}
 	}
 	seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
 		   mappable_count, mappable_size);
 	seq_printf(m, "%u fault mappable objects, %zu bytes\n",
 		   count, size);
 	seq_printf(m, "%zu [%zu] gtt total\n",
 		   dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_gem_gtt_info(struct seq_file *m, void* data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	size_t total_obj_size, total_gtt_size;
 	int count, ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	total_obj_size = total_gtt_size = count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 		seq_printf(m, "   ");
 		describe_obj(m, obj);
 		seq_printf(m, "\n");
 		total_obj_size += obj->base.size;
 		total_gtt_size += obj->gtt_space->size;
 		count++;
 	}
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n",
 		   count, total_obj_size, total_gtt_size);
 	return 0;
 }
 static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	unsigned long flags;
 	struct intel_crtc *crtc;
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
 		const char pipe = pipe_name(crtc->pipe);
 		const char plane = plane_name(crtc->plane);
 		struct intel_unpin_work *work;
 		spin_lock_irqsave(&dev->event_lock, flags);
 		work = crtc->unpin_work;
 		if (work == NULL) {
 			seq_printf(m, "No flip due on pipe %c (plane %c)\n",
 				   pipe, plane);
 		} else {
 			if (!work->pending) {
 				seq_printf(m, "Flip queued on pipe %c (plane %c)\n",
 					   pipe, plane);
 			} else {
 				seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
 					   pipe, plane);
 			}
 			if (work->enable_stall_check)
 				seq_printf(m, "Stall check enabled, ");
 			else
 				seq_printf(m, "Stall check waiting for page flip ioctl, ");
 			seq_printf(m, "%d prepares\n", work->pending);
 			if (work->old_fb_obj) {
 				struct drm_i915_gem_object *obj = work->old_fb_obj;
 				if (obj)
 					seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
 			}
 			if (work->pending_flip_obj) {
 				struct drm_i915_gem_object *obj = work->pending_flip_obj;
 				if (obj)
 					seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
 			}
 		}
 		spin_unlock_irqrestore(&dev->event_lock, flags);
 	}
 	return 0;
 }
 static int i915_gem_request_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_request *gem_request;
 	int ret, count;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	count = 0;
 	if (!list_empty(&dev_priv->ring[RCS].request_list)) {
 		seq_printf(m, "Render requests:\n");
 		list_for_each_entry(gem_request,
 				    &dev_priv->ring[RCS].request_list,
 				    list) {
 			seq_printf(m, "    %d @ %d\n",
 				   gem_request->seqno,
 				   (int) (jiffies - gem_request->emitted_jiffies));
 		}
 		count++;
 	}
 	if (!list_empty(&dev_priv->ring[VCS].request_list)) {
 		seq_printf(m, "BSD requests:\n");
 		list_for_each_entry(gem_request,
 				    &dev_priv->ring[VCS].request_list,
 				    list) {
 			seq_printf(m, "    %d @ %d\n",
 				   gem_request->seqno,
 				   (int) (jiffies - gem_request->emitted_jiffies));
 		}
 		count++;
 	}
 	if (!list_empty(&dev_priv->ring[BCS].request_list)) {
 		seq_printf(m, "BLT requests:\n");
 		list_for_each_entry(gem_request,
 				    &dev_priv->ring[BCS].request_list,
 				    list) {
 			seq_printf(m, "    %d @ %d\n",
 				   gem_request->seqno,
 				   (int) (jiffies - gem_request->emitted_jiffies));
 		}
 		count++;
 	}
 	mutex_unlock(&dev->struct_mutex);
 	if (count == 0)
 		seq_printf(m, "No requests\n");
 	return 0;
 }
 static void i915_ring_seqno_info(struct seq_file *m,
 				 struct intel_ring_buffer *ring)
 {
 	if (ring->get_seqno) {
 		seq_printf(m, "Current sequence (%s): %d\n",
 			   ring->name, ring->get_seqno(ring));
 		seq_printf(m, "Waiter sequence (%s):  %d\n",
 			   ring->name, ring->waiting_seqno);
 		seq_printf(m, "IRQ sequence (%s):     %d\n",
 			   ring->name, ring->irq_seqno);
 	}
 }
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret, i;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	for (i = 0; i < I915_NUM_RINGS; i++)
 		i915_ring_seqno_info(m, &dev_priv->ring[i]);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_interrupt_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret, i, pipe;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	if (!HAS_PCH_SPLIT(dev)) {
 		seq_printf(m, "Interrupt enable:    %08x\n",
 			   I915_READ(IER));
 		seq_printf(m, "Interrupt identity:  %08x\n",
 			   I915_READ(IIR));
 		seq_printf(m, "Interrupt mask:      %08x\n",
 			   I915_READ(IMR));
 		for_each_pipe(pipe)
 			seq_printf(m, "Pipe %c stat:         %08x\n",
 				   pipe_name(pipe),
 				   I915_READ(PIPESTAT(pipe)));
 	} else {
 		seq_printf(m, "North Display Interrupt enable:		%08x\n",
 			   I915_READ(DEIER));
 		seq_printf(m, "North Display Interrupt identity:	%08x\n",
 			   I915_READ(DEIIR));
 		seq_printf(m, "North Display Interrupt mask:		%08x\n",
 			   I915_READ(DEIMR));
 		seq_printf(m, "South Display Interrupt enable:		%08x\n",
 			   I915_READ(SDEIER));
 		seq_printf(m, "South Display Interrupt identity:	%08x\n",
 			   I915_READ(SDEIIR));
 		seq_printf(m, "South Display Interrupt mask:		%08x\n",
 			   I915_READ(SDEIMR));
 		seq_printf(m, "Graphics Interrupt enable:		%08x\n",
 			   I915_READ(GTIER));
 		seq_printf(m, "Graphics Interrupt identity:		%08x\n",
 			   I915_READ(GTIIR));
 		seq_printf(m, "Graphics Interrupt mask:		%08x\n",
 			   I915_READ(GTIMR));
 	}
 	seq_printf(m, "Interrupts received: %d\n",
 		   atomic_read(&dev_priv->irq_received));
 	for (i = 0; i < I915_NUM_RINGS; i++) {
 		if (IS_GEN6(dev) || IS_GEN7(dev)) {
 			seq_printf(m, "Graphics Interrupt mask (%s):	%08x\n",
 				   dev_priv->ring[i].name,
 				   I915_READ_IMR(&dev_priv->ring[i]));
 		}
 		i915_ring_seqno_info(m, &dev_priv->ring[i]);
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int i, ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
 	seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
 		seq_printf(m, "Fenced object[%2d] = ", i);
 		if (obj == NULL)
 			seq_printf(m, "unused");
 		else
 			describe_obj(m, obj);
 		seq_printf(m, "\n");
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_hws_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;
 	const volatile u32 __iomem *hws;
 	int i;
 	ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
 	hws = (volatile u32 __iomem *)ring->status_page.page_addr;
 	if (hws == NULL)
 		return 0;
 	for (i = 0; i < 4096 / sizeof(u32) / 4; i += 4) {
 		seq_printf(m, "0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
 			   i * 4,
 			   hws[i], hws[i + 1], hws[i + 2], hws[i + 3]);
 	}
 	return 0;
 }
 static void i915_dump_object(struct seq_file *m,
 			     struct io_mapping *mapping,
 			     struct drm_i915_gem_object *obj)
 {
 	int page, page_count, i;
 	page_count = obj->base.size / PAGE_SIZE;
 	for (page = 0; page < page_count; page++) {
 		u32 *mem = io_mapping_map_wc(mapping,
 					     obj->gtt_offset + page * PAGE_SIZE);
 		for (i = 0; i < PAGE_SIZE; i += 4)
 			seq_printf(m, "%08x :  %08x\n", i, mem[i / 4]);
 		io_mapping_unmap(mem);
 	}
 }
 static int i915_batchbuffer_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
 		if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) {
 		    seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
 		    i915_dump_object(m, dev_priv->mm.gtt_mapping, obj);
 		}
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_ringbuffer_data(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
 	if (!ring->obj) {
 		seq_printf(m, "No ringbuffer setup\n");
 	} else {
 		const u8 __iomem *virt = ring->virtual_start;
 		uint32_t off;
 		for (off = 0; off < ring->size; off += 4) {
 			uint32_t *ptr = (uint32_t *)(virt + off);
 			seq_printf(m, "%08x :  %08x\n", off, *ptr);
 		}
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_ringbuffer_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;
 	int ret;
 	ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
 	if (ring->size == 0)
 		return 0;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	seq_printf(m, "Ring %s:\n", ring->name);
 	seq_printf(m, "  Head :    %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
 	seq_printf(m, "  Tail :    %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
 	seq_printf(m, "  Size :    %08x\n", ring->size);
 	seq_printf(m, "  Active :  %08x\n", intel_ring_get_active_head(ring));
 	seq_printf(m, "  NOPID :   %08x\n", I915_READ_NOPID(ring));
 	if (IS_GEN6(dev)) {
 		seq_printf(m, "  Sync 0 :   %08x\n", I915_READ_SYNC_0(ring));
 		seq_printf(m, "  Sync 1 :   %08x\n", I915_READ_SYNC_1(ring));
 	}
 	seq_printf(m, "  Control : %08x\n", I915_READ_CTL(ring));
 	seq_printf(m, "  Start :   %08x\n", I915_READ_START(ring));
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static const char *ring_str(int ring)
 {
 	switch (ring) {
 	case RING_RENDER: return " render";
 	case RING_BSD: return " bsd";
 	case RING_BLT: return " blt";
 	default: return "";
 	}
 }
 static const char *pin_flag(int pinned)
 {
 	if (pinned > 0)
 		return " P";
 	else if (pinned < 0)
 		return " p";
 	else
 		return "";
 }
 static const char *tiling_flag(int tiling)
 {
 	switch (tiling) {
 	default:
 	case I915_TILING_NONE: return "";
 	case I915_TILING_X: return " X";
 	case I915_TILING_Y: return " Y";
 	}
 }
 static const char *dirty_flag(int dirty)
 {
 	return dirty ? " dirty" : "";
 }
 static const char *purgeable_flag(int purgeable)
 {
 	return purgeable ? " purgeable" : "";
 }
 static void print_error_buffers(struct seq_file *m,
 				const char *name,
 				struct drm_i915_error_buffer *err,
 				int count)
 {
 	seq_printf(m, "%s [%d]:\n", name, count);
 	while (count--) {
 		seq_printf(m, "  %08x %8u %04x %04x %08x%s%s%s%s%s%s",
 			   err->gtt_offset,
 			   err->size,
 			   err->read_domains,
 			   err->write_domain,
 			   err->seqno,
 			   pin_flag(err->pinned),
 			   tiling_flag(err->tiling),
 			   dirty_flag(err->dirty),
 			   purgeable_flag(err->purgeable),
 			   ring_str(err->ring),
 			   cache_level_str(err->cache_level));
 		if (err->name)
 			seq_printf(m, " (name: %d)", err->name);
 		if (err->fence_reg != I915_FENCE_REG_NONE)
 			seq_printf(m, " (fence: %d)", err->fence_reg);
 		seq_printf(m, "\n");
 		err++;
 	}
 }
 static int i915_error_state(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_error_state *error;
 	unsigned long flags;
 	int i, page, offset, elt;
 	spin_lock_irqsave(&dev_priv->error_lock, flags);
 	if (!dev_priv->first_error) {
 		seq_printf(m, "no error state collected\n");
 		goto out;
 	}
 	error = dev_priv->first_error;
 	seq_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
 		   error->time.tv_usec);
 	seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
 	seq_printf(m, "EIR: 0x%08x\n", error->eir);
 	seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
 	if (INTEL_INFO(dev)->gen >= 6) {
 		seq_printf(m, "ERROR: 0x%08x\n", error->error);
 		seq_printf(m, "Blitter command stream:\n");
 		seq_printf(m, "  ACTHD:    0x%08x\n", error->bcs_acthd);
 		seq_printf(m, "  IPEIR:    0x%08x\n", error->bcs_ipeir);
 		seq_printf(m, "  IPEHR:    0x%08x\n", error->bcs_ipehr);
 		seq_printf(m, "  INSTDONE: 0x%08x\n", error->bcs_instdone);
 		seq_printf(m, "  seqno:    0x%08x\n", error->bcs_seqno);
 		seq_printf(m, "Video (BSD) command stream:\n");
 		seq_printf(m, "  ACTHD:    0x%08x\n", error->vcs_acthd);
 		seq_printf(m, "  IPEIR:    0x%08x\n", error->vcs_ipeir);
 		seq_printf(m, "  IPEHR:    0x%08x\n", error->vcs_ipehr);
 		seq_printf(m, "  INSTDONE: 0x%08x\n", error->vcs_instdone);
 		seq_printf(m, "  seqno:    0x%08x\n", error->vcs_seqno);
 	}
 	seq_printf(m, "Render command stream:\n");
 	seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd);
 	seq_printf(m, "  IPEIR: 0x%08x\n", error->ipeir);
 	seq_printf(m, "  IPEHR: 0x%08x\n", error->ipehr);
 	seq_printf(m, "  INSTDONE: 0x%08x\n", error->instdone);
 	if (INTEL_INFO(dev)->gen >= 4) {
 		seq_printf(m, "  INSTDONE1: 0x%08x\n", error->instdone1);
 		seq_printf(m, "  INSTPS: 0x%08x\n", error->instps);
 	}
 	seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm);
 	seq_printf(m, "  seqno: 0x%08x\n", error->seqno);
 	for (i = 0; i < dev_priv->num_fence_regs; i++)
 		seq_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
 	if (error->active_bo)
 		print_error_buffers(m, "Active",
 				    error->active_bo,
 				    error->active_bo_count);
 	if (error->pinned_bo)
 		print_error_buffers(m, "Pinned",
 				    error->pinned_bo,
 				    error->pinned_bo_count);
 	for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) {
 		if (error->batchbuffer[i]) {
 			struct drm_i915_error_object *obj = error->batchbuffer[i];
 			seq_printf(m, "%s --- gtt_offset = 0x%08x\n",
 				   dev_priv->ring[i].name,
 				   obj->gtt_offset);
 			offset = 0;
 			for (page = 0; page < obj->page_count; page++) {
 				for (elt = 0; elt < PAGE_SIZE/4; elt++) {
 					seq_printf(m, "%08x :  %08x\n", offset, obj->pages[page][elt]);
 					offset += 4;
 				}
 			}
 		}
 	}
 	for (i = 0; i < ARRAY_SIZE(error->ringbuffer); i++) {
 		if (error->ringbuffer[i]) {
 			struct drm_i915_error_object *obj = error->ringbuffer[i];
 			seq_printf(m, "%s --- ringbuffer = 0x%08x\n",
 				   dev_priv->ring[i].name,
 				   obj->gtt_offset);
 			offset = 0;
 			for (page = 0; page < obj->page_count; page++) {
 				for (elt = 0; elt < PAGE_SIZE/4; elt++) {
 					seq_printf(m, "%08x :  %08x\n",
 						   offset,
 						   obj->pages[page][elt]);
 					offset += 4;
 				}
 			}
 		}
 	}
 	if (error->overlay)
 		intel_overlay_print_error_state(m, error->overlay);
 	if (error->display)
 		intel_display_print_error_state(m, dev, error->display);
 out:
 	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 	return 0;
 }
 static int i915_rstdby_delays(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u16 crstanddelay;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	crstanddelay = I915_READ16(CRSTANDVID);
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "w/ctx: %d, w/o ctx: %d\n", (crstanddelay >> 8) & 0x3f, (crstanddelay & 0x3f));
 	return 0;
 }
 static int i915_cur_delayinfo(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret;
 	if (IS_GEN5(dev)) {
 		u16 rgvswctl = I915_READ16(MEMSWCTL);
 		u16 rgvstat = I915_READ16(MEMSTAT_ILK);
 		seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
 		seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
 		seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
 			   MEMSTAT_VID_SHIFT);
 		seq_printf(m, "Current P-state: %d\n",
 			   (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
 	} else if (IS_GEN6(dev) || IS_GEN7(dev)) {
 		u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
 		u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
 		u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
 		u32 rpstat;
 		u32 rpupei, rpcurup, rpprevup;
 		u32 rpdownei, rpcurdown, rpprevdown;
 		int max_freq;
 		/* RPSTAT1 is in the GT power well */
 		ret = mutex_lock_interruptible(&dev->struct_mutex);
 		if (ret)
 			return ret;
 		gen6_gt_force_wake_get(dev_priv);
 		rpstat = I915_READ(GEN6_RPSTAT1);
 		rpupei = I915_READ(GEN6_RP_CUR_UP_EI);
 		rpcurup = I915_READ(GEN6_RP_CUR_UP);
 		rpprevup = I915_READ(GEN6_RP_PREV_UP);
 		rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI);
 		rpcurdown = I915_READ(GEN6_RP_CUR_DOWN);
 		rpprevdown = I915_READ(GEN6_RP_PREV_DOWN);
 		gen6_gt_force_wake_put(dev_priv);
 		mutex_unlock(&dev->struct_mutex);
 		seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
 		seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat);
 		seq_printf(m, "Render p-state ratio: %d\n",
 			   (gt_perf_status & 0xff00) >> 8);
 		seq_printf(m, "Render p-state VID: %d\n",
 			   gt_perf_status & 0xff);
 		seq_printf(m, "Render p-state limit: %d\n",
 			   rp_state_limits & 0xff);
 		seq_printf(m, "CAGF: %dMHz\n", ((rpstat & GEN6_CAGF_MASK) >>
 						GEN6_CAGF_SHIFT) * 50);
 		seq_printf(m, "RP CUR UP EI: %dus\n", rpupei &
 			   GEN6_CURICONT_MASK);
 		seq_printf(m, "RP CUR UP: %dus\n", rpcurup &
 			   GEN6_CURBSYTAVG_MASK);
 		seq_printf(m, "RP PREV UP: %dus\n", rpprevup &
 			   GEN6_CURBSYTAVG_MASK);
 		seq_printf(m, "RP CUR DOWN EI: %dus\n", rpdownei &
 			   GEN6_CURIAVG_MASK);
 		seq_printf(m, "RP CUR DOWN: %dus\n", rpcurdown &
 			   GEN6_CURBSYTAVG_MASK);
 		seq_printf(m, "RP PREV DOWN: %dus\n", rpprevdown &
 			   GEN6_CURBSYTAVG_MASK);
 		max_freq = (rp_state_cap & 0xff0000) >> 16;
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
 			   max_freq * 50);
 		max_freq = (rp_state_cap & 0xff00) >> 8;
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
 			   max_freq * 50);
 		max_freq = rp_state_cap & 0xff;
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
 			   max_freq * 50);
 	} else {
 		seq_printf(m, "no P-state info available\n");
 	}
 	return 0;
 }
 static int i915_delayfreq_table(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 delayfreq;
 	int ret, i;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	for (i = 0; i < 16; i++) {
 		delayfreq = I915_READ(PXVFREQ_BASE + i * 4);
 		seq_printf(m, "P%02dVIDFREQ: 0x%08x (VID: %d)\n", i, delayfreq,
 			   (delayfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT);
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static inline int MAP_TO_MV(int map)
 {
 	return 1250 - (map * 25);
 }
 static int i915_inttoext_table(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 inttoext;
 	int ret, i;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	for (i = 1; i <= 32; i++) {
 		inttoext = I915_READ(INTTOEXT_BASE_ILK + i * 4);
 		seq_printf(m, "INTTOEXT%02d: 0x%08x\n", i, inttoext);
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int ironlake_drpc_info(struct seq_file *m)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 rgvmodectl, rstdbyctl;
 	u16 crstandvid;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	rgvmodectl = I915_READ(MEMMODECTL);
 	rstdbyctl = I915_READ(RSTDBYCTL);
 	crstandvid = I915_READ16(CRSTANDVID);
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ?
 		   "yes" : "no");
 	seq_printf(m, "Boost freq: %d\n",
 		   (rgvmodectl & MEMMODE_BOOST_FREQ_MASK) >>
 		   MEMMODE_BOOST_FREQ_SHIFT);
 	seq_printf(m, "HW control enabled: %s\n",
 		   rgvmodectl & MEMMODE_HWIDLE_EN ? "yes" : "no");
 	seq_printf(m, "SW control enabled: %s\n",
 		   rgvmodectl & MEMMODE_SWMODE_EN ? "yes" : "no");
 	seq_printf(m, "Gated voltage change: %s\n",
 		   rgvmodectl & MEMMODE_RCLK_GATE ? "yes" : "no");
 	seq_printf(m, "Starting frequency: P%d\n",
 		   (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT);
 	seq_printf(m, "Max P-state: P%d\n",
 		   (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT);
 	seq_printf(m, "Min P-state: P%d\n", (rgvmodectl & MEMMODE_FMIN_MASK));
 	seq_printf(m, "RS1 VID: %d\n", (crstandvid & 0x3f));
 	seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f));
 	seq_printf(m, "Render standby enabled: %s\n",
 		   (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes");
 	seq_printf(m, "Current RS state: ");
 	switch (rstdbyctl & RSX_STATUS_MASK) {
 	case RSX_STATUS_ON:
 		seq_printf(m, "on\n");
 		break;
 	case RSX_STATUS_RC1:
 		seq_printf(m, "RC1\n");
 		break;
 	case RSX_STATUS_RC1E:
 		seq_printf(m, "RC1E\n");
 		break;
 	case RSX_STATUS_RS1:
 		seq_printf(m, "RS1\n");
 		break;
 	case RSX_STATUS_RS2:
 		seq_printf(m, "RS2 (RC6)\n");
 		break;
 	case RSX_STATUS_RS3:
 		seq_printf(m, "RC3 (RC6+)\n");
 		break;
 	default:
 		seq_printf(m, "unknown\n");
 		break;
 	}
 	return 0;
 }
 static int gen6_drpc_info(struct seq_file *m)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 rpmodectl1, gt_core_status, rcctl1;
 	int count=0, ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	if (atomic_read(&dev_priv->forcewake_count)) {
 		seq_printf(m, "RC information inaccurate because userspace "
 			      "holds a reference \n");
 	} else {
 		/* NB: we cannot use forcewake, else we read the wrong values */
 		while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
 			udelay(10);
 		seq_printf(m, "RC information accurate: %s\n", yesno(count < 51));
 	}
 	gt_core_status = readl(dev_priv->regs + GEN6_GT_CORE_STATUS);
 	trace_i915_reg_rw(false, GEN6_GT_CORE_STATUS, gt_core_status, 4);
 	rpmodectl1 = I915_READ(GEN6_RP_CONTROL);
 	rcctl1 = I915_READ(GEN6_RC_CONTROL);
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "Video Turbo Mode: %s\n",
 		   yesno(rpmodectl1 & GEN6_RP_MEDIA_TURBO));
 	seq_printf(m, "HW control enabled: %s\n",
 		   yesno(rpmodectl1 & GEN6_RP_ENABLE));
 	seq_printf(m, "SW control enabled: %s\n",
 		   yesno((rpmodectl1 & GEN6_RP_MEDIA_MODE_MASK) ==
 			  GEN6_RP_MEDIA_SW_MODE));
 	seq_printf(m, "RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE));
 	seq_printf(m, "RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC6_ENABLE));
 	seq_printf(m, "Deep RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC6p_ENABLE));
 	seq_printf(m, "Deepest RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC6pp_ENABLE));
 	seq_printf(m, "Current RC state: ");
 	switch (gt_core_status & GEN6_RCn_MASK) {
 	case GEN6_RC0:
 		if (gt_core_status & GEN6_CORE_CPD_STATE_MASK)
 			seq_printf(m, "Core Power Down\n");
 		else
 			seq_printf(m, "on\n");
 		break;
 	case GEN6_RC3:
 		seq_printf(m, "RC3\n");
 		break;
 	case GEN6_RC6:
 		seq_printf(m, "RC6\n");
 		break;
 	case GEN6_RC7:
 		seq_printf(m, "RC7\n");
 		break;
 	default:
 		seq_printf(m, "Unknown\n");
 		break;
 	}
 	seq_printf(m, "Core Power Down: %s\n",
 		   yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK));
 	return 0;
 }
 static int i915_drpc_info(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	if (IS_GEN6(dev) || IS_GEN7(dev))
 		return gen6_drpc_info(m);
 	else
 		return ironlake_drpc_info(m);
 }
 static int i915_fbc_status(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	if (!I915_HAS_FBC(dev)) {
 		seq_printf(m, "FBC unsupported on this chipset\n");
 		return 0;
 	}
 	if (intel_fbc_enabled(dev)) {
 		seq_printf(m, "FBC enabled\n");
 	} else {
 		seq_printf(m, "FBC disabled: ");
 		switch (dev_priv->no_fbc_reason) {
 		case FBC_NO_OUTPUT:
 			seq_printf(m, "no outputs");
 			break;
 		case FBC_STOLEN_TOO_SMALL:
 			seq_printf(m, "not enough stolen memory");
 			break;
 		case FBC_UNSUPPORTED_MODE:
 			seq_printf(m, "mode not supported");
 			break;
 		case FBC_MODE_TOO_LARGE:
 			seq_printf(m, "mode too large");
 			break;
 		case FBC_BAD_PLANE:
 			seq_printf(m, "FBC unsupported on plane");
 			break;
 		case FBC_NOT_TILED:
 			seq_printf(m, "scanout buffer not tiled");
 			break;
 		case FBC_MULTIPLE_PIPES:
 			seq_printf(m, "multiple pipes are enabled");
 			break;
 		case FBC_MODULE_PARAM:
 			seq_printf(m, "disabled per module param (default off)");
 			break;
 		default:
 			seq_printf(m, "unknown reason");
 		}
 		seq_printf(m, "\n");
 	}
 	return 0;
 }
 static int i915_sr_status(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	bool sr_enabled = false;
 	if (HAS_PCH_SPLIT(dev))
 		sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
 	else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev))
 		sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 	else if (IS_I915GM(dev))
 		sr_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 	else if (IS_PINEVIEW(dev))
 		sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN;
 	seq_printf(m, "self-refresh: %s\n",
 		   sr_enabled ? "enabled" : "disabled");
 	return 0;
 }
 static int i915_emon_status(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	unsigned long temp, chipset, gfx;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	temp = i915_mch_val(dev_priv);
 	chipset = i915_chipset_val(dev_priv);
 	gfx = i915_gfx_val(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 	seq_printf(m, "GMCH temp: %ld\n", temp);
 	seq_printf(m, "Chipset power: %ld\n", chipset);
 	seq_printf(m, "GFX power: %ld\n", gfx);
 	seq_printf(m, "Total power: %ld\n", chipset + gfx);
 	return 0;
 }
 static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret;
 	int gpu_freq, ia_freq;
 	if (!(IS_GEN6(dev) || IS_GEN7(dev))) {
 		seq_printf(m, "unsupported on this chipset\n");
 		return 0;
 	}
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	seq_printf(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\n");
 	for (gpu_freq = dev_priv->min_delay; gpu_freq <= dev_priv->max_delay;
 	     gpu_freq++) {
 		I915_WRITE(GEN6_PCODE_DATA, gpu_freq);
 		I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
 			   GEN6_PCODE_READ_MIN_FREQ_TABLE);
 		if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
 			      GEN6_PCODE_READY) == 0, 10)) {
 			DRM_ERROR("pcode read of freq table timed out\n");
 			continue;
 		}
 		ia_freq = I915_READ(GEN6_PCODE_DATA);
 		seq_printf(m, "%d\t\t%d\n", gpu_freq * 50, ia_freq * 100);
 	}
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_gfxec(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	seq_printf(m, "GFXEC: %ld\n", (unsigned long)I915_READ(0x112f4));
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_opregion(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_opregion *opregion = &dev_priv->opregion;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	if (opregion->header)
 		seq_write(m, opregion->header, OPREGION_SIZE);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_fbdev *ifbdev;
 	struct intel_framebuffer *fb;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->mode_config.mutex);
 	if (ret)
 		return ret;
 	ifbdev = dev_priv->fbdev;
 	fb = to_intel_framebuffer(ifbdev->helper.fb);
 	seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, obj ",
 		   fb->base.width,
 		   fb->base.height,
 		   fb->base.depth,
 		   fb->base.bits_per_pixel);
 	describe_obj(m, fb->obj);
 	seq_printf(m, "\n");
 	list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
 		if (&fb->base == ifbdev->helper.fb)
 			continue;
 		seq_printf(m, "user size: %d x %d, depth %d, %d bpp, obj ",
 			   fb->base.width,
 			   fb->base.height,
 			   fb->base.depth,
 			   fb->base.bits_per_pixel);
 		describe_obj(m, fb->obj);
 		seq_printf(m, "\n");
 	}
 	mutex_unlock(&dev->mode_config.mutex);
 	return 0;
 }
 static int i915_context_status(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret;
 	ret = mutex_lock_interruptible(&dev->mode_config.mutex);
 	if (ret)
 		return ret;
 	if (dev_priv->pwrctx) {
 		seq_printf(m, "power context ");
 		describe_obj(m, dev_priv->pwrctx);
 		seq_printf(m, "\n");
 	}
 	if (dev_priv->renderctx) {
 		seq_printf(m, "render context ");
 		describe_obj(m, dev_priv->renderctx);
 		seq_printf(m, "\n");
 	}
 	mutex_unlock(&dev->mode_config.mutex);
 	return 0;
 }
 static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	seq_printf(m, "forcewake count = %d\n",
 		   atomic_read(&dev_priv->forcewake_count));
 	return 0;
 }
 static int
 i915_wedged_open(struct inode *inode,
 		 struct file *filp)
 {
 	filp->private_data = inode->i_private;
 	return 0;
 }
 static ssize_t
 i915_wedged_read(struct file *filp,
 		 char __user *ubuf,
 		 size_t max,
 		 loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
 	int len;
 	len = snprintf(buf, sizeof(buf),
 		       "wedged :  %d\n",
 		       atomic_read(&dev_priv->mm.wedged));
 	if (len > sizeof(buf))
 		len = sizeof(buf);
 	return simple_read_from_buffer(ubuf, max, ppos, buf, len);
 }
 static ssize_t
 i915_wedged_write(struct file *filp,
 		  const char __user *ubuf,
 		  size_t cnt,
 		  loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	char buf[20];
 	int val = 1;
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
 			return -EINVAL;
 		if (copy_from_user(buf, ubuf, cnt))
 			return -EFAULT;
 		buf[cnt] = 0;
 		val = simple_strtoul(buf, NULL, 0);
 	}
 	DRM_INFO("Manually setting wedged to %d\n", val);
 	i915_handle_error(dev, val);
 	return cnt;
 }
 static const struct file_operations i915_wedged_fops = {
 	.owner = THIS_MODULE,
 	.open = i915_wedged_open,
 	.read = i915_wedged_read,
 	.write = i915_wedged_write,
 	.llseek = default_llseek,
 };
 static int
 i915_max_freq_open(struct inode *inode,
 		   struct file *filp)
 {
 	filp->private_data = inode->i_private;
 	return 0;
 }
 static ssize_t
 i915_max_freq_read(struct file *filp,
 		   char __user *ubuf,
 		   size_t max,
 		   loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
 	int len;
 	len = snprintf(buf, sizeof(buf),
 		       "max freq: %d\n", dev_priv->max_delay * 50);
 	if (len > sizeof(buf))
 		len = sizeof(buf);
 	return simple_read_from_buffer(ubuf, max, ppos, buf, len);
 }
 static ssize_t
 i915_max_freq_write(struct file *filp,
 		  const char __user *ubuf,
 		  size_t cnt,
 		  loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	char buf[20];
 	int val = 1;
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
 			return -EINVAL;
 		if (copy_from_user(buf, ubuf, cnt))
 			return -EFAULT;
 		buf[cnt] = 0;
 		val = simple_strtoul(buf, NULL, 0);
 	}
 	DRM_DEBUG_DRIVER("Manually setting max freq to %d\n", val);
 	/*
 	 * Turbo will still be enabled, but won't go above the set value.
 	 */
 	dev_priv->max_delay = val / 50;
 	gen6_set_rps(dev, val / 50);
 	return cnt;
 }
 static const struct file_operations i915_max_freq_fops = {
 	.owner = THIS_MODULE,
 	.open = i915_max_freq_open,
 	.read = i915_max_freq_read,
 	.write = i915_max_freq_write,
 	.llseek = default_llseek,
 };
 static int
 i915_cache_sharing_open(struct inode *inode,
 		   struct file *filp)
 {
 	filp->private_data = inode->i_private;
 	return 0;
 }
 static ssize_t
 i915_cache_sharing_read(struct file *filp,
 		   char __user *ubuf,
 		   size_t max,
 		   loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
 	u32 snpcr;
 	int len;
 	mutex_lock(&dev_priv->dev->struct_mutex);
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 	mutex_unlock(&dev_priv->dev->struct_mutex);
 	len = snprintf(buf, sizeof(buf),
 		       "%d\n", (snpcr & GEN6_MBC_SNPCR_MASK) >>
 		       GEN6_MBC_SNPCR_SHIFT);
 	if (len > sizeof(buf))
 		len = sizeof(buf);
 	return simple_read_from_buffer(ubuf, max, ppos, buf, len);
 }
 static ssize_t
 i915_cache_sharing_write(struct file *filp,
 		  const char __user *ubuf,
 		  size_t cnt,
 		  loff_t *ppos)
 {
 	struct drm_device *dev = filp->private_data;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	char buf[20];
 	u32 snpcr;
 	int val = 1;
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
 			return -EINVAL;
 		if (copy_from_user(buf, ubuf, cnt))
 			return -EFAULT;
 		buf[cnt] = 0;
 		val = simple_strtoul(buf, NULL, 0);
 	}
 	if (val < 0 || val > 3)
 		return -EINVAL;
 	DRM_DEBUG_DRIVER("Manually setting uncore sharing to %d\n", val);
 	/* Update the cache sharing policy here as well */
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 	snpcr &= ~GEN6_MBC_SNPCR_MASK;
 	snpcr |= (val << GEN6_MBC_SNPCR_SHIFT);
 	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
 	return cnt;
 }
 static const struct file_operations i915_cache_sharing_fops = {
 	.owner = THIS_MODULE,
 	.open = i915_cache_sharing_open,
 	.read = i915_cache_sharing_read,
 	.write = i915_cache_sharing_write,
 	.llseek = default_llseek,
 };
 /* As the drm_debugfs_init() routines are called before dev->dev_private is
  * allocated we need to hook into the minor for release. */
 static int
 drm_add_fake_info_node(struct drm_minor *minor,
 		       struct dentry *ent,
 		       const void *key)
 {
 	struct drm_info_node *node;
 	node = kmalloc(sizeof(struct drm_info_node), GFP_KERNEL);
 	if (node == NULL) {
 		debugfs_remove(ent);
 		return -ENOMEM;
 	}
 	node->minor = minor;
 	node->dent = ent;
 	node->info_ent = (void *) key;
 	mutex_lock(&minor->debugfs_lock);
 	list_add(&node->list, &minor->debugfs_list);
 	mutex_unlock(&minor->debugfs_lock);
 	return 0;
 }
 static int i915_wedged_create(struct dentry *root, struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 	ent = debugfs_create_file("i915_wedged",
 				  S_IRUGO | S_IWUSR,
 				  root, dev,
 				  &i915_wedged_fops);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 	return drm_add_fake_info_node(minor, ent, &i915_wedged_fops);
 }
 static int i915_forcewake_open(struct inode *inode, struct file *file)
 {
 	struct drm_device *dev = inode->i_private;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 	if (!IS_GEN6(dev))
 		return 0;
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	gen6_gt_force_wake_get(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 int i915_forcewake_release(struct inode *inode, struct file *file)
 {
 	struct drm_device *dev = inode->i_private;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	if (!IS_GEN6(dev))
 		return 0;
 	/*
 	 * It's bad that we can potentially hang userspace if struct_mutex gets
 	 * forever stuck.  However, if we cannot acquire this lock it means that
 	 * almost certainly the driver has hung, is not unload-able. Therefore
 	 * hanging here is probably a minor inconvenience not to be seen my
 	 * almost every user.
 	 */
 	mutex_lock(&dev->struct_mutex);
 	gen6_gt_force_wake_put(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
 }
 static const struct file_operations i915_forcewake_fops = {
 	.owner = THIS_MODULE,
 	.open = i915_forcewake_open,
 	.release = i915_forcewake_release,
 };
 static int i915_forcewake_create(struct dentry *root, struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 	ent = debugfs_create_file("i915_forcewake_user",
 				  S_IRUSR,
 				  root, dev,
 				  &i915_forcewake_fops);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 	return drm_add_fake_info_node(minor, ent, &i915_forcewake_fops);
 }
 static int i915_max_freq_create(struct dentry *root, struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 	ent = debugfs_create_file("i915_max_freq",
 				  S_IRUGO | S_IWUSR,
 				  root, dev,
 				  &i915_max_freq_fops);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 	return drm_add_fake_info_node(minor, ent, &i915_max_freq_fops);
 }
 static int i915_cache_sharing_create(struct dentry *root, struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 	ent = debugfs_create_file("i915_cache_sharing",
 				  S_IRUGO | S_IWUSR,
 				  root, dev,
 				  &i915_cache_sharing_fops);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 	return drm_add_fake_info_node(minor, ent, &i915_cache_sharing_fops);
 }
 static struct drm_info_list i915_debugfs_list[] = {
 	{"i915_capabilities", i915_capabilities, 0},
 	{"i915_gem_objects", i915_gem_object_info, 0},
 	{"i915_gem_gtt", i915_gem_gtt_info, 0},
 	{"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST},
 	{"i915_gem_flushing", i915_gem_object_list_info, 0, (void *) FLUSHING_LIST},
 	{"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST},
 	{"i915_gem_pinned", i915_gem_object_list_info, 0, (void *) PINNED_LIST},
 	{"i915_gem_deferred_free", i915_gem_object_list_info, 0, (void *) DEFERRED_FREE_LIST},
 	{"i915_gem_pageflip", i915_gem_pageflip_info, 0},
 	{"i915_gem_request", i915_gem_request_info, 0},
 	{"i915_gem_seqno", i915_gem_seqno_info, 0},
 	{"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
 	{"i915_gem_interrupt", i915_interrupt_info, 0},
 	{"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
 	{"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
 	{"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
 	{"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
 	{"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
 	{"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
 	{"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
 	{"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
 	{"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
 	{"i915_batchbuffers", i915_batchbuffer_info, 0},
 	{"i915_error_state", i915_error_state, 0},
 	{"i915_rstdby_delays", i915_rstdby_delays, 0},
 	{"i915_cur_delayinfo", i915_cur_delayinfo, 0},
 	{"i915_delayfreq_table", i915_delayfreq_table, 0},
 	{"i915_inttoext_table", i915_inttoext_table, 0},
 	{"i915_drpc_info", i915_drpc_info, 0},
 	{"i915_emon_status", i915_emon_status, 0},
 	{"i915_ring_freq_table", i915_ring_freq_table, 0},
 	{"i915_gfxec", i915_gfxec, 0},
 	{"i915_fbc_status", i915_fbc_status, 0},
 	{"i915_sr_status", i915_sr_status, 0},
 	{"i915_opregion", i915_opregion, 0},
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
 	{"i915_context_status", i915_context_status, 0},
 	{"i915_gen6_forcewake_count", i915_gen6_forcewake_count_info, 0},
 };
 #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
 int i915_debugfs_init(struct drm_minor *minor)
 {
 	int ret;
 	ret = i915_wedged_create(minor->debugfs_root, minor);
 	if (ret)
 		return ret;
 	ret = i915_forcewake_create(minor->debugfs_root, minor);
 	if (ret)
 		return ret;
 	ret = i915_max_freq_create(minor->debugfs_root, minor);
 	if (ret)
 		return ret;
 	ret = i915_cache_sharing_create(minor->debugfs_root, minor);
 	if (ret)
 		return ret;
 	return drm_debugfs_create_files(i915_debugfs_list,
 					I915_DEBUGFS_ENTRIES,
 					minor->debugfs_root, minor);
 }
 void i915_debugfs_cleanup(struct drm_minor *minor)
 {
 	drm_debugfs_remove_files(i915_debugfs_list,
 				 I915_DEBUGFS_ENTRIES, minor);
 	drm_debugfs_remove_files((struct drm_info_list *) &i915_forcewake_fops,
 				 1, minor);
 	drm_debugfs_remove_files((struct drm_info_list *) &i915_wedged_fops,
 				 1, minor);
 	drm_debugfs_remove_files((struct drm_info_list *) &i915_max_freq_fops,
 				 1, minor);
 	drm_debugfs_remove_files((struct drm_info_list *) &i915_cache_sharing_fops,
 				 1, minor);
 }
 #endif /* CONFIG_DEBUG_FS */

drivers/gpu/drm/i915/i915_dma.c

Diff comments View file @ 3d29b84

 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
  */
 /*
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * without limitation the rights to use, copy, modify, merge, publish,
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
 #include "drmP.h"
 #include "drm.h"
 #include "drm_crtc_helper.h"
 #include "drm_fb_helper.h"
 #include "intel_drv.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "../../../platform/x86/intel_ips.h"
 #include <linux/pci.h>
 #include <linux/vgaarb.h>
 #include <linux/acpi.h>
 #include <linux/pnp.h>
 #include <linux/vga_switcheroo.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <acpi/video.h>
 static void i915_write_hws_pga(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 addr;
 	addr = dev_priv->status_page_dmah->busaddr;
 	if (INTEL_INFO(dev)->gen >= 4)
 		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 	I915_WRITE(HWS_PGA, addr);
 }
 /**
  * Sets up the hardware status page for devices that need a physical address
  * in the register.
  */
 static int i915_init_phys_hws(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	/* Program Hardware Status Page */
 	dev_priv->status_page_dmah =
 		drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE);
 	if (!dev_priv->status_page_dmah) {
 		DRM_ERROR("Can not allocate hardware status page\n");
 		return -ENOMEM;
 	}
 	memset_io((void __force __iomem *)dev_priv->status_page_dmah->vaddr,
 		  0, PAGE_SIZE);
 	i915_write_hws_pga(dev);
 	DRM_DEBUG_DRIVER("Enabled hardware status page\n");
 	return 0;
 }
 /**
  * Frees the hardware status page, whether it's a physical address or a virtual
  * address set up by the X Server.
  */
 static void i915_free_hws(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = LP_RING(dev_priv);
 	if (dev_priv->status_page_dmah) {
 		drm_pci_free(dev, dev_priv->status_page_dmah);
 		dev_priv->status_page_dmah = NULL;
 	}
 	if (ring->status_page.gfx_addr) {
 		ring->status_page.gfx_addr = 0;
 		drm_core_ioremapfree(&dev_priv->hws_map, dev);
 	}
 	/* Need to rewrite hardware status page */
 	I915_WRITE(HWS_PGA, 0x1ffff000);
 }
 void i915_kernel_lost_context(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv;
 	struct intel_ring_buffer *ring = LP_RING(dev_priv);
 	/*
 	 * We should never lose context on the ring with modesetting
 	 * as we don't expose it to userspace
 	 */
 	if (drm_core_check_feature(dev, DRIVER_MODESET))
 		return;
 	ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 	ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 	ring->space = ring->head - (ring->tail + 8);
 	if (ring->space < 0)
 		ring->space += ring->size;
 	if (!dev->primary->master)
 		return;
 	master_priv = dev->primary->master->driver_priv;
 	if (ring->head == ring->tail && master_priv->sarea_priv)
 		master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY;
 }
 static int i915_dma_cleanup(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int i;
 	/* Make sure interrupts are disabled here because the uninstall ioctl
 	 * may not have been called from userspace and after dev_private
 	 * is freed, it's too late.
 	 */
 	if (dev->irq_enabled)
 		drm_irq_uninstall(dev);
 	mutex_lock(&dev->struct_mutex);
 	for (i = 0; i < I915_NUM_RINGS; i++)
 		intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 	mutex_unlock(&dev->struct_mutex);
 	/* Clear the HWS virtual address at teardown */
 	if (I915_NEED_GFX_HWS(dev))
 		i915_free_hws(dev);
 	return 0;
 }
 static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 	int ret;
 	master_priv->sarea = drm_getsarea(dev);
 	if (master_priv->sarea) {
 		master_priv->sarea_priv = (drm_i915_sarea_t *)
 			((u8 *)master_priv->sarea->handle + init->sarea_priv_offset);
 	} else {
 		DRM_DEBUG_DRIVER("sarea not found assuming DRI2 userspace\n");
 	}
 	if (init->ring_size != 0) {
 		if (LP_RING(dev_priv)->obj != NULL) {
 			i915_dma_cleanup(dev);
 			DRM_ERROR("Client tried to initialize ringbuffer in "
 				  "GEM mode\n");
 			return -EINVAL;
 		}
 		ret = intel_render_ring_init_dri(dev,
 						 init->ring_start,
 						 init->ring_size);
 		if (ret) {
 			i915_dma_cleanup(dev);
 			return ret;
 		}
 	}
 	dev_priv->cpp = init->cpp;
 	dev_priv->back_offset = init->back_offset;
 	dev_priv->front_offset = init->front_offset;
 	dev_priv->current_page = 0;
 	if (master_priv->sarea_priv)
 		master_priv->sarea_priv->pf_current_page = 0;
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
 	dev_priv->allow_batchbuffer = 1;
 	return 0;
 }
 static int i915_dma_resume(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	struct intel_ring_buffer *ring = LP_RING(dev_priv);
 	DRM_DEBUG_DRIVER("%s\n", __func__);
 	if (ring->map.handle == NULL) {
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
 		return -ENOMEM;
 	}
 	/* Program Hardware Status Page */
 	if (!ring->status_page.page_addr) {
 		DRM_ERROR("Can not find hardware status page\n");
 		return -EINVAL;
 	}
 	DRM_DEBUG_DRIVER("hw status page @ %p\n",
 				ring->status_page.page_addr);
 	if (ring->status_page.gfx_addr != 0)
 		intel_ring_setup_status_page(ring);
 	else
 		i915_write_hws_pga(dev);
 	DRM_DEBUG_DRIVER("Enabled hardware status page\n");
 	return 0;
 }
 static int i915_dma_init(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv)
 {
 	drm_i915_init_t *init = data;
 	int retcode = 0;
 	switch (init->func) {
 	case I915_INIT_DMA:
 		retcode = i915_initialize(dev, init);
 		break;
 	case I915_CLEANUP_DMA:
 		retcode = i915_dma_cleanup(dev);
 		break;
 	case I915_RESUME_DMA:
 		retcode = i915_dma_resume(dev);
 		break;
 	default:
 		retcode = -EINVAL;
 		break;
 	}
 	return retcode;
 }
 /* Implement basically the same security restrictions as hardware does
  * for MI_BATCH_NON_SECURE.  These can be made stricter at any time.
  *
  * Most of the calculations below involve calculating the size of a
  * particular instruction.  It's important to get the size right as
  * that tells us where the next instruction to check is.  Any illegal
  * instruction detected will be given a size of zero, which is a
  * signal to abort the rest of the buffer.
  */
 static int validate_cmd(int cmd)
 {
 	switch (((cmd >> 29) & 0x7)) {
 	case 0x0:
 		switch ((cmd >> 23) & 0x3f) {
 		case 0x0:
 			return 1;	/* MI_NOOP */
 		case 0x4:
 			return 1;	/* MI_FLUSH */
 		default:
 			return 0;	/* disallow everything else */
 		}
 		break;
 	case 0x1:
 		return 0;	/* reserved */
 	case 0x2:
 		return (cmd & 0xff) + 2;	/* 2d commands */
 	case 0x3:
 		if (((cmd >> 24) & 0x1f) <= 0x18)
 			return 1;
 		switch ((cmd >> 24) & 0x1f) {
 		case 0x1c:
 			return 1;
 		case 0x1d:
 			switch ((cmd >> 16) & 0xff) {
 			case 0x3:
 				return (cmd & 0x1f) + 2;
 			case 0x4:
 				return (cmd & 0xf) + 2;
 			default:
 				return (cmd & 0xffff) + 2;
 			}
 		case 0x1e:
 			if (cmd & (1 << 23))
 				return (cmd & 0xffff) + 1;
 			else
 				return 1;
 		case 0x1f:
 			if ((cmd & (1 << 23)) == 0)	/* inline vertices */
 				return (cmd & 0x1ffff) + 2;
 			else if (cmd & (1 << 17))	/* indirect random */
 				if ((cmd & 0xffff) == 0)
 					return 0;	/* unknown length, too hard */
 				else
 					return (((cmd & 0xffff) + 1) / 2) + 1;
 			else
 				return 2;	/* indirect sequential */
 		default:
 			return 0;
 		}
 	default:
 		return 0;
 	}
 	return 0;
 }
 static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int i, ret;
 	if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
 		return -EINVAL;
 	for (i = 0; i < dwords;) {
 		int sz = validate_cmd(buffer[i]);
 		if (sz == 0 || i + sz > dwords)
 			return -EINVAL;
 		i += sz;
 	}
 	ret = BEGIN_LP_RING((dwords+1)&~1);
 	if (ret)
 		return ret;
 	for (i = 0; i < dwords; i++)
 		OUT_RING(buffer[i]);
 	if (dwords & 1)
 		OUT_RING(0);
 	ADVANCE_LP_RING();
 	return 0;
 }
 int
 i915_emit_box(struct drm_device *dev,
 	      struct drm_clip_rect *box,
 	      int DR1, int DR4)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 	if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
 	    box->y2 <= 0 || box->x2 <= 0) {
 		DRM_ERROR("Bad box %d,%d..%d,%d\n",
 			  box->x1, box->y1, box->x2, box->y2);
 		return -EINVAL;
 	}
 	if (INTEL_INFO(dev)->gen >= 4) {
 		ret = BEGIN_LP_RING(4);
 		if (ret)
 			return ret;
 		OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
 		OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
 		OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
 		OUT_RING(DR4);
 	} else {
 		ret = BEGIN_LP_RING(6);
 		if (ret)
 			return ret;
 		OUT_RING(GFX_OP_DRAWRECT_INFO);
 		OUT_RING(DR1);
 		OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
 		OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
 		OUT_RING(DR4);
 		OUT_RING(0);
 	}
 	ADVANCE_LP_RING();
 	return 0;
 }
 /* XXX: Emitting the counter should really be moved to part of the IRQ
  * emit. For now, do it in both places:
  */
 static void i915_emit_breadcrumb(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 	dev_priv->counter++;
 	if (dev_priv->counter > 0x7FFFFFFFUL)
 		dev_priv->counter = 0;
 	if (master_priv->sarea_priv)
 		master_priv->sarea_priv->last_enqueue = dev_priv->counter;
 	if (BEGIN_LP_RING(4) == 0) {
 		OUT_RING(MI_STORE_DWORD_INDEX);
 		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 		OUT_RING(dev_priv->counter);
 		OUT_RING(0);
 		ADVANCE_LP_RING();
 	}
 }
 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 				   drm_i915_cmdbuffer_t *cmd,
 				   struct drm_clip_rect *cliprects,
 				   void *cmdbuf)
 {
 	int nbox = cmd->num_cliprects;
 	int i = 0, count, ret;
 	if (cmd->sz & 0x3) {
 		DRM_ERROR("alignment");
 		return -EINVAL;
 	}
 	i915_kernel_lost_context(dev);
 	count = nbox ? nbox : 1;
 	for (i = 0; i < count; i++) {
 		if (i < nbox) {
 			ret = i915_emit_box(dev, &cliprects[i],
 					    cmd->DR1, cmd->DR4);
 			if (ret)
 				return ret;
 		}
 		ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4);
 		if (ret)
 			return ret;
 	}
 	i915_emit_breadcrumb(dev);
 	return 0;
 }
 static int i915_dispatch_batchbuffer(struct drm_device * dev,
 				     drm_i915_batchbuffer_t * batch,
 				     struct drm_clip_rect *cliprects)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int nbox = batch->num_cliprects;
 	int i, count, ret;
 	if ((batch->start | batch->used) & 0x7) {
 		DRM_ERROR("alignment");
 		return -EINVAL;
 	}
 	i915_kernel_lost_context(dev);
 	count = nbox ? nbox : 1;
 	for (i = 0; i < count; i++) {
 		if (i < nbox) {
 			ret = i915_emit_box(dev, &cliprects[i],
 					    batch->DR1, batch->DR4);
 			if (ret)
 				return ret;
 		}
 		if (!IS_I830(dev) && !IS_845G(dev)) {
 			ret = BEGIN_LP_RING(2);
 			if (ret)
 				return ret;
 			if (INTEL_INFO(dev)->gen >= 4) {
 				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
 				OUT_RING(batch->start);
 			} else {
 				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
 				OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 			}
 		} else {
 			ret = BEGIN_LP_RING(4);
 			if (ret)
 				return ret;
 			OUT_RING(MI_BATCH_BUFFER);
 			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 			OUT_RING(batch->start + batch->used - 4);
 			OUT_RING(0);
 		}
 		ADVANCE_LP_RING();
 	}
 	if (IS_G4X(dev) || IS_GEN5(dev)) {
 		if (BEGIN_LP_RING(2) == 0) {
 			OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
 			OUT_RING(MI_NOOP);
 			ADVANCE_LP_RING();
 		}
 	}
 	i915_emit_breadcrumb(dev);
 	return 0;
 }
 static int i915_dispatch_flip(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv =
 		dev->primary->master->driver_priv;
 	int ret;
 	if (!master_priv->sarea_priv)
 		return -EINVAL;
 	DRM_DEBUG_DRIVER("%s: page=%d pfCurrentPage=%d\n",
 			  __func__,
 			 dev_priv->current_page,
 			 master_priv->sarea_priv->pf_current_page);
 	i915_kernel_lost_context(dev);
 	ret = BEGIN_LP_RING(10);
 	if (ret)
 		return ret;
 	OUT_RING(MI_FLUSH | MI_READ_FLUSH);
 	OUT_RING(0);
 	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
 	OUT_RING(0);
 	if (dev_priv->current_page == 0) {
 		OUT_RING(dev_priv->back_offset);
 		dev_priv->current_page = 1;
 	} else {
 		OUT_RING(dev_priv->front_offset);
 		dev_priv->current_page = 0;
 	}
 	OUT_RING(0);
 	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
 	OUT_RING(0);
 	ADVANCE_LP_RING();
 	master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
 	if (BEGIN_LP_RING(4) == 0) {
 		OUT_RING(MI_STORE_DWORD_INDEX);
 		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 		OUT_RING(dev_priv->counter);
 		OUT_RING(0);
 		ADVANCE_LP_RING();
 	}
 	master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
 	return 0;
 }
 static int i915_quiescent(struct drm_device *dev)
 {
 	struct intel_ring_buffer *ring = LP_RING(dev->dev_private);
 	i915_kernel_lost_context(dev);
 	return intel_wait_ring_idle(ring);
 }
 static int i915_flush_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv)
 {
 	int ret;
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_quiescent(dev);
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
 }
 static int i915_batchbuffer(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    master_priv->sarea_priv;
 	drm_i915_batchbuffer_t *batch = data;
 	int ret;
 	struct drm_clip_rect *cliprects = NULL;
 	if (!dev_priv->allow_batchbuffer) {
 		DRM_ERROR("Batchbuffer ioctl disabled\n");
 		return -EINVAL;
 	}
 	DRM_DEBUG_DRIVER("i915 batchbuffer, start %x used %d cliprects %d\n",
 			batch->start, batch->used, batch->num_cliprects);
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 	if (batch->num_cliprects < 0)
 		return -EINVAL;
 	if (batch->num_cliprects) {
 		cliprects = kcalloc(batch->num_cliprects,
 				    sizeof(struct drm_clip_rect),
 				    GFP_KERNEL);
 		if (cliprects == NULL)
 			return -ENOMEM;
 		ret = copy_from_user(cliprects, batch->cliprects,
 				     batch->num_cliprects *
 				     sizeof(struct drm_clip_rect));
 		if (ret != 0) {
 			ret = -EFAULT;
 			goto fail_free;
 		}
 	}
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_dispatch_batchbuffer(dev, batch, cliprects);
 	mutex_unlock(&dev->struct_mutex);
 	if (sarea_priv)
 		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 fail_free:
 	kfree(cliprects);
 	return ret;
 }
 static int i915_cmdbuffer(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    master_priv->sarea_priv;
 	drm_i915_cmdbuffer_t *cmdbuf = data;
 	struct drm_clip_rect *cliprects = NULL;
 	void *batch_data;
 	int ret;
 	DRM_DEBUG_DRIVER("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
 			cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 	if (cmdbuf->num_cliprects < 0)
 		return -EINVAL;
 	batch_data = kmalloc(cmdbuf->sz, GFP_KERNEL);
 	if (batch_data == NULL)
 		return -ENOMEM;
 	ret = copy_from_user(batch_data, cmdbuf->buf, cmdbuf->sz);
 	if (ret != 0) {
 		ret = -EFAULT;
 		goto fail_batch_free;
 	}
 	if (cmdbuf->num_cliprects) {
 		cliprects = kcalloc(cmdbuf->num_cliprects,
 				    sizeof(struct drm_clip_rect), GFP_KERNEL);
 		if (cliprects == NULL) {
 			ret = -ENOMEM;
 			goto fail_batch_free;
 		}
 		ret = copy_from_user(cliprects, cmdbuf->cliprects,
 				     cmdbuf->num_cliprects *
 				     sizeof(struct drm_clip_rect));
 		if (ret != 0) {
 			ret = -EFAULT;
 			goto fail_clip_free;
 		}
 	}
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data);
 	mutex_unlock(&dev->struct_mutex);
 	if (ret) {
 		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
 		goto fail_clip_free;
 	}
 	if (sarea_priv)
 		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 fail_clip_free:
 	kfree(cliprects);
 fail_batch_free:
 	kfree(batch_data);
 	return ret;
 }
 static int i915_flip_bufs(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
 	int ret;
 	DRM_DEBUG_DRIVER("%s\n", __func__);
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_dispatch_flip(dev);
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
 }
 static int i915_getparam(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_getparam_t *param = data;
 	int value;
 	if (!dev_priv) {
 		DRM_ERROR("called with no initialization\n");
 		return -EINVAL;
 	}
 	switch (param->param) {
 	case I915_PARAM_IRQ_ACTIVE:
 		value = dev->pdev->irq ? 1 : 0;
 		break;
 	case I915_PARAM_ALLOW_BATCHBUFFER:
 		value = dev_priv->allow_batchbuffer ? 1 : 0;
 		break;
 	case I915_PARAM_LAST_DISPATCH:
 		value = READ_BREADCRUMB(dev_priv);
 		break;
 	case I915_PARAM_CHIPSET_ID:
 		value = dev->pci_device;
 		break;
 	case I915_PARAM_HAS_GEM:
 		value = dev_priv->has_gem;
 		break;
 	case I915_PARAM_NUM_FENCES_AVAIL:
 		value = dev_priv->num_fence_regs - dev_priv->fence_reg_start;
 		break;
 	case I915_PARAM_HAS_OVERLAY:
 		value = dev_priv->overlay ? 1 : 0;
 		break;
 	case I915_PARAM_HAS_PAGEFLIPPING:
 		value = 1;
 		break;
 	case I915_PARAM_HAS_EXECBUF2:
 		/* depends on GEM */
 		value = dev_priv->has_gem;
 		break;
 	case I915_PARAM_HAS_BSD:
 		value = HAS_BSD(dev);
 		break;
 	case I915_PARAM_HAS_BLT:
 		value = HAS_BLT(dev);
 		break;
 	case I915_PARAM_HAS_RELAXED_FENCING:
 		value = 1;
 		break;
 	case I915_PARAM_HAS_COHERENT_RINGS:
 		value = 1;
 		break;
 	case I915_PARAM_HAS_EXEC_CONSTANTS:
 		value = INTEL_INFO(dev)->gen >= 4;
 		break;
 	case I915_PARAM_HAS_RELAXED_DELTA:
 		value = 1;
 		break;
 	case I915_PARAM_HAS_GEN7_SOL_RESET:
 		value = 1;
 		break;
+	case I915_PARAM_HAS_LLC:
+		value = HAS_LLC(dev);
+		break;
 	default:
 		DRM_DEBUG_DRIVER("Unknown parameter %d\n",
 				 param->param);
 		return -EINVAL;
 	}
 	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("DRM_COPY_TO_USER failed\n");
 		return -EFAULT;
 	}
 	return 0;
 }
 static int i915_setparam(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_setparam_t *param = data;
 	if (!dev_priv) {
 		DRM_ERROR("called with no initialization\n");
 		return -EINVAL;
 	}
 	switch (param->param) {
 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
 		break;
 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
 		dev_priv->tex_lru_log_granularity = param->value;
 		break;
 	case I915_SETPARAM_ALLOW_BATCHBUFFER:
 		dev_priv->allow_batchbuffer = param->value;
 		break;
 	case I915_SETPARAM_NUM_USED_FENCES:
 		if (param->value > dev_priv->num_fence_regs ||
 		    param->value < 0)
 			return -EINVAL;
 		/* Userspace can use first N regs */
 		dev_priv->fence_reg_start = param->value;
 		break;
 	default:
 		DRM_DEBUG_DRIVER("unknown parameter %d\n",
 					param->param);
 		return -EINVAL;
 	}
 	return 0;
 }
 static int i915_set_status_page(struct drm_device *dev, void *data,
 				struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_hws_addr_t *hws = data;
 	struct intel_ring_buffer *ring = LP_RING(dev_priv);
 	if (!I915_NEED_GFX_HWS(dev))
 		return -EINVAL;
 	if (!dev_priv) {
 		DRM_ERROR("called with no initialization\n");
 		return -EINVAL;
 	}
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		WARN(1, "tried to set status page when mode setting active\n");
 		return 0;
 	}
 	DRM_DEBUG_DRIVER("set status page addr 0x%08x\n", (u32)hws->addr);
 	ring->status_page.gfx_addr = hws->addr & (0x1ffff<<12);
 	dev_priv->hws_map.offset = dev->agp->base + hws->addr;
 	dev_priv->hws_map.size = 4*1024;
 	dev_priv->hws_map.type = 0;
 	dev_priv->hws_map.flags = 0;
 	dev_priv->hws_map.mtrr = 0;
 	drm_core_ioremap_wc(&dev_priv->hws_map, dev);
 	if (dev_priv->hws_map.handle == NULL) {
 		i915_dma_cleanup(dev);
 		ring->status_page.gfx_addr = 0;
 		DRM_ERROR("can not ioremap virtual address for"
 				" G33 hw status page\n");
 		return -ENOMEM;
 	}
 	ring->status_page.page_addr =
 		(void __force __iomem *)dev_priv->hws_map.handle;
 	memset_io(ring->status_page.page_addr, 0, PAGE_SIZE);
 	I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 	DRM_DEBUG_DRIVER("load hws HWS_PGA with gfx mem 0x%x\n",
 			 ring->status_page.gfx_addr);
 	DRM_DEBUG_DRIVER("load hws at %p\n",
 			 ring->status_page.page_addr);
 	return 0;
 }
 static int i915_get_bridge_dev(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
 	if (!dev_priv->bridge_dev) {
 		DRM_ERROR("bridge device not found\n");
 		return -1;
 	}
 	return 0;
 }
 #define MCHBAR_I915 0x44
 #define MCHBAR_I965 0x48
 #define MCHBAR_SIZE (4*4096)
 #define DEVEN_REG 0x54
 #define   DEVEN_MCHBAR_EN (1 << 28)
 /* Allocate space for the MCH regs if needed, return nonzero on error */
 static int
 intel_alloc_mchbar_resource(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp_lo, temp_hi = 0;
 	u64 mchbar_addr;
 	int ret;
 	if (INTEL_INFO(dev)->gen >= 4)
 		pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
 	pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
 	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
 	/* If ACPI doesn't have it, assume we need to allocate it ourselves */
 #ifdef CONFIG_PNP
 	if (mchbar_addr &&
 	    pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
 		return 0;
 #endif
 	/* Get some space for it */
 	dev_priv->mch_res.name = "i915 MCHBAR";
 	dev_priv->mch_res.flags = IORESOURCE_MEM;
 	ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
 				     &dev_priv->mch_res,
 				     MCHBAR_SIZE, MCHBAR_SIZE,
 				     PCIBIOS_MIN_MEM,
 				     0, pcibios_align_resource,
 				     dev_priv->bridge_dev);
 	if (ret) {
 		DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
 		dev_priv->mch_res.start = 0;
 		return ret;
 	}
 	if (INTEL_INFO(dev)->gen >= 4)
 		pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
 				       upper_32_bits(dev_priv->mch_res.start));
 	pci_write_config_dword(dev_priv->bridge_dev, reg,
 			       lower_32_bits(dev_priv->mch_res.start));
 	return 0;
 }
 /* Setup MCHBAR if possible, return true if we should disable it again */
 static void
 intel_setup_mchbar(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp;
 	bool enabled;
 	dev_priv->mchbar_need_disable = false;
 	if (IS_I915G(dev) || IS_I915GM(dev)) {
 		pci_read_config_dword(dev_priv->bridge_dev, DEVEN_REG, &temp);
 		enabled = !!(temp & DEVEN_MCHBAR_EN);
 	} else {
 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
 		enabled = temp & 1;
 	}
 	/* If it's already enabled, don't have to do anything */
 	if (enabled)
 		return;
 	if (intel_alloc_mchbar_resource(dev))
 		return;
 	dev_priv->mchbar_need_disable = true;
 	/* Space is allocated or reserved, so enable it. */
 	if (IS_I915G(dev) || IS_I915GM(dev)) {
 		pci_write_config_dword(dev_priv->bridge_dev, DEVEN_REG,
 				       temp | DEVEN_MCHBAR_EN);
 	} else {
 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
 		pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
 	}
 }
 static void
 intel_teardown_mchbar(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp;
 	if (dev_priv->mchbar_need_disable) {
 		if (IS_I915G(dev) || IS_I915GM(dev)) {
 			pci_read_config_dword(dev_priv->bridge_dev, DEVEN_REG, &temp);
 			temp &= ~DEVEN_MCHBAR_EN;
 			pci_write_config_dword(dev_priv->bridge_dev, DEVEN_REG, temp);
 		} else {
 			pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
 			temp &= ~1;
 			pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp);
 		}
 	}
 	if (dev_priv->mch_res.start)
 		release_resource(&dev_priv->mch_res);
 }
 #define PTE_ADDRESS_MASK		0xfffff000
 #define PTE_ADDRESS_MASK_HIGH		0x000000f0 /* i915+ */
 #define PTE_MAPPING_TYPE_UNCACHED	(0 << 1)
 #define PTE_MAPPING_TYPE_DCACHE		(1 << 1) /* i830 only */
 #define PTE_MAPPING_TYPE_CACHED		(3 << 1)
 #define PTE_MAPPING_TYPE_MASK		(3 << 1)
 #define PTE_VALID			(1 << 0)
 /**
  * i915_stolen_to_phys - take an offset into stolen memory and turn it into
  *                       a physical one
  * @dev: drm device
  * @offset: address to translate
  *
  * Some chip functions require allocations from stolen space and need the
  * physical address of the memory in question.
  */
 static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct pci_dev *pdev = dev_priv->bridge_dev;
 	u32 base;
 #if 0
 	/* On the machines I have tested the Graphics Base of Stolen Memory
 	 * is unreliable, so compute the base by subtracting the stolen memory
 	 * from the Top of Low Usable DRAM which is where the BIOS places
 	 * the graphics stolen memory.
 	 */
 	if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
 		/* top 32bits are reserved = 0 */
 		pci_read_config_dword(pdev, 0xA4, &base);
 	} else {
 		/* XXX presume 8xx is the same as i915 */
 		pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
 	}
 #else
 	if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
 		u16 val;
 		pci_read_config_word(pdev, 0xb0, &val);
 		base = val >> 4 << 20;
 	} else {
 		u8 val;
 		pci_read_config_byte(pdev, 0x9c, &val);
 		base = val >> 3 << 27;
 	}
 	base -= dev_priv->mm.gtt->stolen_size;
 #endif
 	return base + offset;
 }
 static void i915_warn_stolen(struct drm_device *dev)
 {
 	DRM_ERROR("not enough stolen space for compressed buffer, disabling\n");
 	DRM_ERROR("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
 }
 static void i915_setup_compression(struct drm_device *dev, int size)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_mm_node *compressed_fb, *uninitialized_var(compressed_llb);
 	unsigned long cfb_base;
 	unsigned long ll_base = 0;
 	/* Just in case the BIOS is doing something questionable. */
 	intel_disable_fbc(dev);
 	compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
 	if (compressed_fb)
 		compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
 	if (!compressed_fb)
 		goto err;
 	cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
 	if (!cfb_base)
 		goto err_fb;
 	if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) {
 		compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
 						    4096, 4096, 0);
 		if (compressed_llb)
 			compressed_llb = drm_mm_get_block(compressed_llb,
 							  4096, 4096);
 		if (!compressed_llb)
 			goto err_fb;
 		ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
 		if (!ll_base)
 			goto err_llb;
 	}
 	dev_priv->cfb_size = size;
 	dev_priv->compressed_fb = compressed_fb;
 	if (HAS_PCH_SPLIT(dev))
 		I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
 	else if (IS_GM45(dev)) {
 		I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
 	} else {
 		I915_WRITE(FBC_CFB_BASE, cfb_base);
 		I915_WRITE(FBC_LL_BASE, ll_base);
 		dev_priv->compressed_llb = compressed_llb;
 	}
 	DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
 		      cfb_base, ll_base, size >> 20);
 	return;
 err_llb:
 	drm_mm_put_block(compressed_llb);
 err_fb:
 	drm_mm_put_block(compressed_fb);
 err:
 	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
 	i915_warn_stolen(dev);
 }
 static void i915_cleanup_compression(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	drm_mm_put_block(dev_priv->compressed_fb);
 	if (dev_priv->compressed_llb)
 		drm_mm_put_block(dev_priv->compressed_llb);
 }
 /* true = enable decode, false = disable decoder */
 static unsigned int i915_vga_set_decode(void *cookie, bool state)
 {
 	struct drm_device *dev = cookie;
 	intel_modeset_vga_set_state(dev, state);
 	if (state)
 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 	else
 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 }
 static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
 {
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
 	if (state == VGA_SWITCHEROO_ON) {
 		printk(KERN_INFO "i915: switched on\n");
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 		/* i915 resume handler doesn't set to D0 */
 		pci_set_power_state(dev->pdev, PCI_D0);
 		i915_resume(dev);
 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
 	} else {
 		printk(KERN_ERR "i915: switched off\n");
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 		i915_suspend(dev, pmm);
 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
 	}
 }
 static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
 {
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	bool can_switch;
 	spin_lock(&dev->count_lock);
 	can_switch = (dev->open_count == 0);
 	spin_unlock(&dev->count_lock);
 	return can_switch;
 }
 static int i915_load_gem_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	unsigned long prealloc_size, gtt_size, mappable_size;
 	int ret;
 	prealloc_size = dev_priv->mm.gtt->stolen_size;
 	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
 	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
 	/* Basic memrange allocator for stolen space */
 	drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
 	/* Let GEM Manage all of the aperture.
 	 *
 	 * However, leave one page at the end still bound to the scratch page.
 	 * There are a number of places where the hardware apparently
 	 * prefetches past the end of the object, and we've seen multiple
 	 * hangs with the GPU head pointer stuck in a batchbuffer bound
 	 * at the last page of the aperture.  One page should be enough to
 	 * keep any prefetching inside of the aperture.
 	 */
 	i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_gem_init_ringbuffer(dev);
 	mutex_unlock(&dev->struct_mutex);
 	if (ret)
 		return ret;
 	/* Try to set up FBC with a reasonable compressed buffer size */
 	if (I915_HAS_FBC(dev) && i915_powersave) {
 		int cfb_size;
 		/* Leave 1M for line length buffer & misc. */
 		/* Try to get a 32M buffer... */
 		if (prealloc_size > (36*1024*1024))
 			cfb_size = 32*1024*1024;
 		else /* fall back to 7/8 of the stolen space */
 			cfb_size = prealloc_size * 7 / 8;
 		i915_setup_compression(dev, cfb_size);
 	}
 	/* Allow hardware batchbuffers unless told otherwise. */
 	dev_priv->allow_batchbuffer = 1;
 	return 0;
 }
 static int i915_load_modeset_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 	ret = intel_parse_bios(dev);
 	if (ret)
 		DRM_INFO("failed to find VBIOS tables\n");
 	/* If we have > 1 VGA cards, then we need to arbitrate access
 	 * to the common VGA resources.
 	 *
 	 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
 	 * then we do not take part in VGA arbitration and the
 	 * vga_client_register() fails with -ENODEV.
 	 */
 	ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
 	if (ret && ret != -ENODEV)
 		goto out;
 	intel_register_dsm_handler();
 	ret = vga_switcheroo_register_client(dev->pdev,
 					     i915_switcheroo_set_state,
 					     NULL,
 					     i915_switcheroo_can_switch);
 	if (ret)
 		goto cleanup_vga_client;
 	/* IIR "flip pending" bit means done if this bit is set */
 	if (IS_GEN3(dev) && (I915_READ(ECOSKPD) & ECO_FLIP_DONE))
 		dev_priv->flip_pending_is_done = true;
 	intel_modeset_init(dev);
 	ret = i915_load_gem_init(dev);
 	if (ret)
 		goto cleanup_vga_switcheroo;
 	intel_modeset_gem_init(dev);
 	ret = drm_irq_install(dev);
 	if (ret)
 		goto cleanup_gem;
 	/* Always safe in the mode setting case. */
 	/* FIXME: do pre/post-mode set stuff in core KMS code */
 	dev->vblank_disable_allowed = 1;
 	ret = intel_fbdev_init(dev);
 	if (ret)
 		goto cleanup_irq;
 	drm_kms_helper_poll_init(dev);
 	/* We're off and running w/KMS */
 	dev_priv->mm.suspended = 0;
 	return 0;
 cleanup_irq:
 	drm_irq_uninstall(dev);
 cleanup_gem:
 	mutex_lock(&dev->struct_mutex);
 	i915_gem_cleanup_ringbuffer(dev);
 	mutex_unlock(&dev->struct_mutex);
 cleanup_vga_switcheroo:
 	vga_switcheroo_unregister_client(dev->pdev);
 cleanup_vga_client:
 	vga_client_register(dev->pdev, NULL, NULL, NULL);
 out:
 	return ret;
 }
 int i915_master_create(struct drm_device *dev, struct drm_master *master)
 {
 	struct drm_i915_master_private *master_priv;
 	master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL);
 	if (!master_priv)
 		return -ENOMEM;
 	master->driver_priv = master_priv;
 	return 0;
 }
 void i915_master_destroy(struct drm_device *dev, struct drm_master *master)
 {
 	struct drm_i915_master_private *master_priv = master->driver_priv;
 	if (!master_priv)
 		return;
 	kfree(master_priv);
 	master->driver_priv = NULL;
 }
 static void i915_pineview_get_mem_freq(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 tmp;
 	tmp = I915_READ(CLKCFG);
 	switch (tmp & CLKCFG_FSB_MASK) {
 	case CLKCFG_FSB_533:
 		dev_priv->fsb_freq = 533; /* 133*4 */
 		break;
 	case CLKCFG_FSB_800:
 		dev_priv->fsb_freq = 800; /* 200*4 */
 		break;
 	case CLKCFG_FSB_667:
 		dev_priv->fsb_freq =  667; /* 167*4 */
 		break;
 	case CLKCFG_FSB_400:
 		dev_priv->fsb_freq = 400; /* 100*4 */
 		break;
 	}
 	switch (tmp & CLKCFG_MEM_MASK) {
 	case CLKCFG_MEM_533:
 		dev_priv->mem_freq = 533;
 		break;
 	case CLKCFG_MEM_667:
 		dev_priv->mem_freq = 667;
 		break;
 	case CLKCFG_MEM_800:
 		dev_priv->mem_freq = 800;
 		break;
 	}
 	/* detect pineview DDR3 setting */
 	tmp = I915_READ(CSHRDDR3CTL);
 	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 }
 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u16 ddrpll, csipll;
 	ddrpll = I915_READ16(DDRMPLL1);
 	csipll = I915_READ16(CSIPLL0);
 	switch (ddrpll & 0xff) {
 	case 0xc:
 		dev_priv->mem_freq = 800;
 		break;
 	case 0x10:
 		dev_priv->mem_freq = 1066;
 		break;
 	case 0x14:
 		dev_priv->mem_freq = 1333;
 		break;
 	case 0x18:
 		dev_priv->mem_freq = 1600;
 		break;
 	default:
 		DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 				 ddrpll & 0xff);
 		dev_priv->mem_freq = 0;
 		break;
 	}
 	dev_priv->r_t = dev_priv->mem_freq;
 	switch (csipll & 0x3ff) {
 	case 0x00c:
 		dev_priv->fsb_freq = 3200;
 		break;
 	case 0x00e:
 		dev_priv->fsb_freq = 3733;
 		break;
 	case 0x010:
 		dev_priv->fsb_freq = 4266;
 		break;
 	case 0x012:
 		dev_priv->fsb_freq = 4800;
 		break;
 	case 0x014:
 		dev_priv->fsb_freq = 5333;
 		break;
 	case 0x016:
 		dev_priv->fsb_freq = 5866;
 		break;
 	case 0x018:
 		dev_priv->fsb_freq = 6400;
 		break;
 	default:
 		DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 				 csipll & 0x3ff);
 		dev_priv->fsb_freq = 0;
 		break;
 	}
 	if (dev_priv->fsb_freq == 3200) {
 		dev_priv->c_m = 0;
 	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 		dev_priv->c_m = 1;
 	} else {
 		dev_priv->c_m = 2;
 	}
 }
 static const struct cparams {
 	u16 i;
 	u16 t;
 	u16 m;
 	u16 c;
 } cparams[] = {
 	{ 1, 1333, 301, 28664 },
 	{ 1, 1066, 294, 24460 },
 	{ 1, 800, 294, 25192 },
 	{ 0, 1333, 276, 27605 },
 	{ 0, 1066, 276, 27605 },
 	{ 0, 800, 231, 23784 },
 };
 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 {
 	u64 total_count, diff, ret;
 	u32 count1, count2, count3, m = 0, c = 0;
 	unsigned long now = jiffies_to_msecs(jiffies), diff1;
 	int i;
 	diff1 = now - dev_priv->last_time1;
 	/* Prevent division-by-zero if we are asking too fast.
 	 * Also, we don't get interesting results if we are polling
 	 * faster than once in 10ms, so just return the saved value
 	 * in such cases.
 	 */
 	if (diff1 <= 10)
 		return dev_priv->chipset_power;
 	count1 = I915_READ(DMIEC);
 	count2 = I915_READ(DDREC);
 	count3 = I915_READ(CSIEC);
 	total_count = count1 + count2 + count3;
 	/* FIXME: handle per-counter overflow */
 	if (total_count < dev_priv->last_count1) {
 		diff = ~0UL - dev_priv->last_count1;
 		diff += total_count;
 	} else {
 		diff = total_count - dev_priv->last_count1;
 	}
 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 		if (cparams[i].i == dev_priv->c_m &&
 		    cparams[i].t == dev_priv->r_t) {
 			m = cparams[i].m;
 			c = cparams[i].c;
 			break;
 		}
 	}
 	diff = div_u64(diff, diff1);
 	ret = ((m * diff) + c);
 	ret = div_u64(ret, 10);
 	dev_priv->last_count1 = total_count;
 	dev_priv->last_time1 = now;
 	dev_priv->chipset_power = ret;
 	return ret;
 }
 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
 {
 	unsigned long m, x, b;
 	u32 tsfs;
 	tsfs = I915_READ(TSFS);
 	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
 	x = I915_READ8(TR1);
 	b = tsfs & TSFS_INTR_MASK;
 	return ((m * x) / 127) - b;
 }
 static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 {
 	static const struct v_table {
 		u16 vd; /* in .1 mil */
 		u16 vm; /* in .1 mil */
 	} v_table[] = {
 		{ 0, 0, },
 		{ 375, 0, },
 		{ 500, 0, },
 		{ 625, 0, },
 		{ 750, 0, },
 		{ 875, 0, },
 		{ 1000, 0, },
 		{ 1125, 0, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4125, 3000, },
 		{ 4250, 3125, },
 		{ 4375, 3250, },
 		{ 4500, 3375, },
 		{ 4625, 3500, },
 		{ 4750, 3625, },
 		{ 4875, 3750, },
 		{ 5000, 3875, },
 		{ 5125, 4000, },
 		{ 5250, 4125, },
 		{ 5375, 4250, },
 		{ 5500, 4375, },
 		{ 5625, 4500, },
 		{ 5750, 4625, },
 		{ 5875, 4750, },
 		{ 6000, 4875, },
 		{ 6125, 5000, },
 		{ 6250, 5125, },
 		{ 6375, 5250, },
 		{ 6500, 5375, },
 		{ 6625, 5500, },
 		{ 6750, 5625, },
 		{ 6875, 5750, },
 		{ 7000, 5875, },
 		{ 7125, 6000, },
 		{ 7250, 6125, },
 		{ 7375, 6250, },
 		{ 7500, 6375, },
 		{ 7625, 6500, },
 		{ 7750, 6625, },
 		{ 7875, 6750, },
 		{ 8000, 6875, },
 		{ 8125, 7000, },
 		{ 8250, 7125, },
 		{ 8375, 7250, },
 		{ 8500, 7375, },
 		{ 8625, 7500, },
 		{ 8750, 7625, },
 		{ 8875, 7750, },
 		{ 9000, 7875, },
 		{ 9125, 8000, },
 		{ 9250, 8125, },
 		{ 9375, 8250, },
 		{ 9500, 8375, },
 		{ 9625, 8500, },
 		{ 9750, 8625, },
 		{ 9875, 8750, },
 		{ 10000, 8875, },
 		{ 10125, 9000, },
 		{ 10250, 9125, },
 		{ 10375, 9250, },
 		{ 10500, 9375, },
 		{ 10625, 9500, },
 		{ 10750, 9625, },
 		{ 10875, 9750, },
 		{ 11000, 9875, },
 		{ 11125, 10000, },
 		{ 11250, 10125, },
 		{ 11375, 10250, },
 		{ 11500, 10375, },
 		{ 11625, 10500, },
 		{ 11750, 10625, },
 		{ 11875, 10750, },
 		{ 12000, 10875, },
 		{ 12125, 11000, },
 		{ 12250, 11125, },
 		{ 12375, 11250, },
 		{ 12500, 11375, },
 		{ 12625, 11500, },
 		{ 12750, 11625, },
 		{ 12875, 11750, },
 		{ 13000, 11875, },
 		{ 13125, 12000, },
 		{ 13250, 12125, },
 		{ 13375, 12250, },
 		{ 13500, 12375, },
 		{ 13625, 12500, },
 		{ 13750, 12625, },
 		{ 13875, 12750, },
 		{ 14000, 12875, },
 		{ 14125, 13000, },
 		{ 14250, 13125, },
 		{ 14375, 13250, },
 		{ 14500, 13375, },
 		{ 14625, 13500, },
 		{ 14750, 13625, },
 		{ 14875, 13750, },
 		{ 15000, 13875, },
 		{ 15125, 14000, },
 		{ 15250, 14125, },
 		{ 15375, 14250, },
 		{ 15500, 14375, },
 		{ 15625, 14500, },
 		{ 15750, 14625, },
 		{ 15875, 14750, },
 		{ 16000, 14875, },
 		{ 16125, 15000, },
 	};
 	if (dev_priv->info->is_mobile)
 		return v_table[pxvid].vm;
 	else
 		return v_table[pxvid].vd;
 }
 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
 {
 	struct timespec now, diff1;
 	u64 diff;
 	unsigned long diffms;
 	u32 count;
 	getrawmonotonic(&now);
 	diff1 = timespec_sub(now, dev_priv->last_time2);
 	/* Don't divide by 0 */
 	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
 	if (!diffms)
 		return;
 	count = I915_READ(GFXEC);
 	if (count < dev_priv->last_count2) {
 		diff = ~0UL - dev_priv->last_count2;
 		diff += count;
 	} else {
 		diff = count - dev_priv->last_count2;
 	}
 	dev_priv->last_count2 = count;
 	dev_priv->last_time2 = now;
 	/* More magic constants... */
 	diff = diff * 1181;
 	diff = div_u64(diff, diffms * 10);
 	dev_priv->gfx_power = diff;
 }
 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
 {
 	unsigned long t, corr, state1, corr2, state2;
 	u32 pxvid, ext_v;
 	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->cur_delay * 4));
 	pxvid = (pxvid >> 24) & 0x7f;
 	ext_v = pvid_to_extvid(dev_priv, pxvid);
 	state1 = ext_v;
 	t = i915_mch_val(dev_priv);
 	/* Revel in the empirically derived constants */
 	/* Correction factor in 1/100000 units */
 	if (t > 80)
 		corr = ((t * 2349) + 135940);
 	else if (t >= 50)
 		corr = ((t * 964) + 29317);
 	else /* < 50 */
 		corr = ((t * 301) + 1004);
 	corr = corr * ((150142 * state1) / 10000 - 78642);
 	corr /= 100000;
 	corr2 = (corr * dev_priv->corr);
 	state2 = (corr2 * state1) / 10000;
 	state2 /= 100; /* convert to mW */
 	i915_update_gfx_val(dev_priv);
 	return dev_priv->gfx_power + state2;
 }
 /* Global for IPS driver to get at the current i915 device */
 static struct drm_i915_private *i915_mch_dev;
 /*
  * Lock protecting IPS related data structures
  *   - i915_mch_dev
  *   - dev_priv->max_delay
  *   - dev_priv->min_delay
  *   - dev_priv->fmax
  *   - dev_priv->gpu_busy
  */
 static DEFINE_SPINLOCK(mchdev_lock);
 /**
  * i915_read_mch_val - return value for IPS use
  *
  * Calculate and return a value for the IPS driver to use when deciding whether
  * we have thermal and power headroom to increase CPU or GPU power budget.
  */
 unsigned long i915_read_mch_val(void)
 {
 	struct drm_i915_private *dev_priv;
 	unsigned long chipset_val, graphics_val, ret = 0;
 	spin_lock(&mchdev_lock);
 	if (!i915_mch_dev)
 		goto out_unlock;
 	dev_priv = i915_mch_dev;
 	chipset_val = i915_chipset_val(dev_priv);
 	graphics_val = i915_gfx_val(dev_priv);
 	ret = chipset_val + graphics_val;
 out_unlock:
 	spin_unlock(&mchdev_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(i915_read_mch_val);
 /**
  * i915_gpu_raise - raise GPU frequency limit
  *
  * Raise the limit; IPS indicates we have thermal headroom.
  */
 bool i915_gpu_raise(void)
 {
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 	spin_lock(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 	if (dev_priv->max_delay > dev_priv->fmax)
 		dev_priv->max_delay--;
 out_unlock:
 	spin_unlock(&mchdev_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(i915_gpu_raise);
 /**
  * i915_gpu_lower - lower GPU frequency limit
  *
  * IPS indicates we're close to a thermal limit, so throttle back the GPU
  * frequency maximum.
  */
 bool i915_gpu_lower(void)
 {
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 	spin_lock(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 	if (dev_priv->max_delay < dev_priv->min_delay)
 		dev_priv->max_delay++;
 out_unlock:
 	spin_unlock(&mchdev_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(i915_gpu_lower);
 /**
  * i915_gpu_busy - indicate GPU business to IPS
  *
  * Tell the IPS driver whether or not the GPU is busy.
  */
 bool i915_gpu_busy(void)
 {
 	struct drm_i915_private *dev_priv;
 	bool ret = false;
 	spin_lock(&mchdev_lock);
 	if (!i915_mch_dev)
 		goto out_unlock;
 	dev_priv = i915_mch_dev;
 	ret = dev_priv->busy;
 out_unlock:
 	spin_unlock(&mchdev_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(i915_gpu_busy);
 /**
  * i915_gpu_turbo_disable - disable graphics turbo
  *
  * Disable graphics turbo by resetting the max frequency and setting the
  * current frequency to the default.
  */
 bool i915_gpu_turbo_disable(void)
 {
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 	spin_lock(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 	dev_priv->max_delay = dev_priv->fstart;
 	if (!ironlake_set_drps(dev_priv->dev, dev_priv->fstart))
 		ret = false;
 out_unlock:
 	spin_unlock(&mchdev_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 /**
  * Tells the intel_ips driver that the i915 driver is now loaded, if
  * IPS got loaded first.
  *
  * This awkward dance is so that neither module has to depend on the
  * other in order for IPS to do the appropriate communication of
  * GPU turbo limits to i915.
  */
 static void
 ips_ping_for_i915_load(void)
 {
 	void (*link)(void);
 	link = symbol_get(ips_link_to_i915_driver);
 	if (link) {
 		link();
 		symbol_put(ips_link_to_i915_driver);
 	}
 }
 /**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
  * @flags: startup flags
  *
  * The driver load routine has to do several things:
  *   - drive output discovery via intel_modeset_init()
  *   - initialize the memory manager
  *   - allocate initial config memory
  *   - setup the DRM framebuffer with the allocated memory
  */
 int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
 	struct drm_i915_private *dev_priv;
 	int ret = 0, mmio_bar;
 	uint32_t agp_size;
 	/* i915 has 4 more counters */
 	dev->counters += 4;
 	dev->types[6] = _DRM_STAT_IRQ;
 	dev->types[7] = _DRM_STAT_PRIMARY;
 	dev->types[8] = _DRM_STAT_SECONDARY;
 	dev->types[9] = _DRM_STAT_DMA;
 	dev_priv = kzalloc(sizeof(drm_i915_private_t), GFP_KERNEL);
 	if (dev_priv == NULL)
 		return -ENOMEM;
 	dev->dev_private = (void *)dev_priv;
 	dev_priv->dev = dev;
 	dev_priv->info = (struct intel_device_info *) flags;
 	if (i915_get_bridge_dev(dev)) {
 		ret = -EIO;
 		goto free_priv;
 	}
 	/* overlay on gen2 is broken and can't address above 1G */
 	if (IS_GEN2(dev))
 		dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
 	/* 965GM sometimes incorrectly writes to hardware status page (HWS)
 	 * using 32bit addressing, overwriting memory if HWS is located
 	 * above 4GB.
 	 *
 	 * The documentation also mentions an issue with undefined
 	 * behaviour if any general state is accessed within a page above 4GB,
 	 * which also needs to be handled carefully.
 	 */
 	if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
 		dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
 	mmio_bar = IS_GEN2(dev) ? 1 : 0;
 	dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
 	if (!dev_priv->regs) {
 		DRM_ERROR("failed to map registers\n");
 		ret = -EIO;
 		goto put_bridge;
 	}
 	dev_priv->mm.gtt = intel_gtt_get();
 	if (!dev_priv->mm.gtt) {
 		DRM_ERROR("Failed to initialize GTT\n");
 		ret = -ENODEV;
 		goto out_rmmap;
 	}
 	agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
 	dev_priv->mm.gtt_mapping =
 		io_mapping_create_wc(dev->agp->base, agp_size);
 	if (dev_priv->mm.gtt_mapping == NULL) {
 		ret = -EIO;
 		goto out_rmmap;
 	}
 	/* Set up a WC MTRR for non-PAT systems.  This is more common than
 	 * one would think, because the kernel disables PAT on first
 	 * generation Core chips because WC PAT gets overridden by a UC
 	 * MTRR if present.  Even if a UC MTRR isn't present.
 	 */
 	dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
 					 agp_size,
 					 MTRR_TYPE_WRCOMB, 1);
 	if (dev_priv->mm.gtt_mtrr < 0) {
 		DRM_INFO("MTRR allocation failed.  Graphics "
 			 "performance may suffer.\n");
 	}
 	/* The i915 workqueue is primarily used for batched retirement of
 	 * requests (and thus managing bo) once the task has been completed
 	 * by the GPU. i915_gem_retire_requests() is called directly when we
 	 * need high-priority retirement, such as waiting for an explicit
 	 * bo.
 	 *
 	 * It is also used for periodic low-priority events, such as
 	 * idle-timers and recording error state.
 	 *
 	 * All tasks on the workqueue are expected to acquire the dev mutex
 	 * so there is no point in running more than one instance of the
 	 * workqueue at any time: max_active = 1 and NON_REENTRANT.
 	 */
 	dev_priv->wq = alloc_workqueue("i915",
 				       WQ_UNBOUND | WQ_NON_REENTRANT,
 				       1);
 	if (dev_priv->wq == NULL) {
 		DRM_ERROR("Failed to create our workqueue.\n");
 		ret = -ENOMEM;
 		goto out_mtrrfree;
 	}
 	/* enable GEM by default */
 	dev_priv->has_gem = 1;
 	intel_irq_init(dev);
 	/* Try to make sure MCHBAR is enabled before poking at it */
 	intel_setup_mchbar(dev);
 	intel_setup_gmbus(dev);
 	intel_opregion_setup(dev);
 	/* Make sure the bios did its job and set up vital registers */
 	intel_setup_bios(dev);
 	i915_gem_load(dev);
 	/* Init HWS */
 	if (!I915_NEED_GFX_HWS(dev)) {
 		ret = i915_init_phys_hws(dev);
 		if (ret)
 			goto out_gem_unload;
 	}
 	if (IS_PINEVIEW(dev))
 		i915_pineview_get_mem_freq(dev);
 	else if (IS_GEN5(dev))
 		i915_ironlake_get_mem_freq(dev);
 	/* On the 945G/GM, the chipset reports the MSI capability on the
 	 * integrated graphics even though the support isn't actually there
 	 * according to the published specs.  It doesn't appear to function
 	 * correctly in testing on 945G.
 	 * This may be a side effect of MSI having been made available for PEG
 	 * and the registers being closely associated.
 	 *
 	 * According to chipset errata, on the 965GM, MSI interrupts may
 	 * be lost or delayed, but we use them anyways to avoid
 	 * stuck interrupts on some machines.
 	 */
 	if (!IS_I945G(dev) && !IS_I945GM(dev))
 		pci_enable_msi(dev->pdev);
 	spin_lock_init(&dev_priv->irq_lock);
 	spin_lock_init(&dev_priv->error_lock);
 	spin_lock_init(&dev_priv->rps_lock);
 	if (IS_IVYBRIDGE(dev))
 		dev_priv->num_pipe = 3;
 	else if (IS_MOBILE(dev) || !IS_GEN2(dev))
 		dev_priv->num_pipe = 2;
 	else
 		dev_priv->num_pipe = 1;
 	ret = drm_vblank_init(dev, dev_priv->num_pipe);
 	if (ret)
 		goto out_gem_unload;
 	/* Start out suspended */
 	dev_priv->mm.suspended = 1;
 	intel_detect_pch(dev);
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		ret = i915_load_modeset_init(dev);
 		if (ret < 0) {
 			DRM_ERROR("failed to init modeset\n");
 			goto out_gem_unload;
 		}
 	}
 	/* Must be done after probing outputs */
 	intel_opregion_init(dev);
 	acpi_video_register();
 	setup_timer(&dev_priv->hangcheck_timer, i915_hangcheck_elapsed,
 		    (unsigned long) dev);
 	spin_lock(&mchdev_lock);
 	i915_mch_dev = dev_priv;
 	dev_priv->mchdev_lock = &mchdev_lock;
 	spin_unlock(&mchdev_lock);
 	ips_ping_for_i915_load();
 	return 0;
 out_gem_unload:
 	if (dev_priv->mm.inactive_shrinker.shrink)
 		unregister_shrinker(&dev_priv->mm.inactive_shrinker);
 	if (dev->pdev->msi_enabled)
 		pci_disable_msi(dev->pdev);
 	intel_teardown_gmbus(dev);
 	intel_teardown_mchbar(dev);
 	destroy_workqueue(dev_priv->wq);
 out_mtrrfree:
 	if (dev_priv->mm.gtt_mtrr >= 0) {
 		mtrr_del(dev_priv->mm.gtt_mtrr, dev->agp->base,
 			 dev->agp->agp_info.aper_size * 1024 * 1024);
 		dev_priv->mm.gtt_mtrr = -1;
 	}
 	io_mapping_free(dev_priv->mm.gtt_mapping);
 out_rmmap:
 	pci_iounmap(dev->pdev, dev_priv->regs);
 put_bridge:
 	pci_dev_put(dev_priv->bridge_dev);
 free_priv:
 	kfree(dev_priv);
 	return ret;
 }
 int i915_driver_unload(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 	spin_lock(&mchdev_lock);
 	i915_mch_dev = NULL;
 	spin_unlock(&mchdev_lock);
 	if (dev_priv->mm.inactive_shrinker.shrink)
 		unregister_shrinker(&dev_priv->mm.inactive_shrinker);
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_gpu_idle(dev);
 	if (ret)
 		DRM_ERROR("failed to idle hardware: %d\n", ret);
 	mutex_unlock(&dev->struct_mutex);
 	/* Cancel the retire work handler, which should be idle now. */
 	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 	io_mapping_free(dev_priv->mm.gtt_mapping);
 	if (dev_priv->mm.gtt_mtrr >= 0) {
 		mtrr_del(dev_priv->mm.gtt_mtrr, dev->agp->base,
 			 dev->agp->agp_info.aper_size * 1024 * 1024);
 		dev_priv->mm.gtt_mtrr = -1;
 	}
 	acpi_video_unregister();
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		intel_fbdev_fini(dev);
 		intel_modeset_cleanup(dev);
 		/*
 		 * free the memory space allocated for the child device
 		 * config parsed from VBT
 		 */
 		if (dev_priv->child_dev && dev_priv->child_dev_num) {
 			kfree(dev_priv->child_dev);
 			dev_priv->child_dev = NULL;
 			dev_priv->child_dev_num = 0;
 		}
 		vga_switcheroo_unregister_client(dev->pdev);
 		vga_client_register(dev->pdev, NULL, NULL, NULL);
 	}
 	/* Free error state after interrupts are fully disabled. */
 	del_timer_sync(&dev_priv->hangcheck_timer);
 	cancel_work_sync(&dev_priv->error_work);
 	i915_destroy_error_state(dev);
 	if (dev->pdev->msi_enabled)
 		pci_disable_msi(dev->pdev);
 	intel_opregion_fini(dev);
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		/* Flush any outstanding unpin_work. */
 		flush_workqueue(dev_priv->wq);
 		mutex_lock(&dev->struct_mutex);
 		i915_gem_free_all_phys_object(dev);
 		i915_gem_cleanup_ringbuffer(dev);
 		mutex_unlock(&dev->struct_mutex);
 		if (I915_HAS_FBC(dev) && i915_powersave)
 			i915_cleanup_compression(dev);
 		drm_mm_takedown(&dev_priv->mm.stolen);
 		intel_cleanup_overlay(dev);
 		if (!I915_NEED_GFX_HWS(dev))
 			i915_free_hws(dev);
 	}
 	if (dev_priv->regs != NULL)
 		pci_iounmap(dev->pdev, dev_priv->regs);
 	intel_teardown_gmbus(dev);
 	intel_teardown_mchbar(dev);
 	destroy_workqueue(dev_priv->wq);
 	pci_dev_put(dev_priv->bridge_dev);
 	kfree(dev->dev_private);
 	return 0;
 }
 int i915_driver_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv;
 	DRM_DEBUG_DRIVER("\n");
 	file_priv = kmalloc(sizeof(*file_priv), GFP_KERNEL);
 	if (!file_priv)
 		return -ENOMEM;
 	file->driver_priv = file_priv;
 	spin_lock_init(&file_priv->mm.lock);
 	INIT_LIST_HEAD(&file_priv->mm.request_list);
 	return 0;
 }
 /**
  * i915_driver_lastclose - clean up after all DRM clients have exited
  * @dev: DRM device
  *
  * Take care of cleaning up after all DRM clients have exited.  In the
  * mode setting case, we want to restore the kernel's initial mode (just
  * in case the last client left us in a bad state).
  *
  * Additionally, in the non-mode setting case, we'll tear down the AGP
  * and DMA structures, since the kernel won't be using them, and clea
  * up any GEM state.
  */
 void i915_driver_lastclose(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	if (!dev_priv || drm_core_check_feature(dev, DRIVER_MODESET)) {
 		intel_fb_restore_mode(dev);
 		vga_switcheroo_process_delayed_switch();
 		return;
 	}
 	i915_gem_lastclose(dev);
 	i915_dma_cleanup(dev);
 }
 void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
 {
 	i915_gem_release(dev, file_priv);
 }
 void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	kfree(file_priv);
 }
 struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_FLIP, i915_flip_bufs, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  i915_vblank_pipe_get, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, intel_sprite_get_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 };
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
 /**
  * Determine if the device really is AGP or not.
  *
  * All Intel graphics chipsets are treated as AGP, even if they are really
  * PCI-e.
  *
  * \param dev   The device to be tested.
  *
  * \returns
  * A value of 1 is always retured to indictate every i9x5 is AGP.
  */
 int i915_driver_device_is_agp(struct drm_device * dev)
 {
 	return 1;
 }

drivers/gpu/drm/i915/i915_drv.c

Diff comments View file @ 3d29b84

 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
  */
 /*
  *
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * without limitation the rights to use, copy, modify, merge, publish,
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
 #include <linux/device.h>
 #include "drmP.h"
 #include "drm.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 #include "intel_drv.h"
 #include <linux/console.h>
 #include <linux/module.h>
 #include "drm_crtc_helper.h"
 static int i915_modeset __read_mostly = -1;
 module_param_named(modeset, i915_modeset, int, 0400);
 MODULE_PARM_DESC(modeset,
 		"Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, "
 		"1=on, -1=force vga console preference [default])");
 unsigned int i915_fbpercrtc __always_unused = 0;
 module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400);
 int i915_panel_ignore_lid __read_mostly = 0;
 module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600);
 MODULE_PARM_DESC(panel_ignore_lid,
 		"Override lid status (0=autodetect [default], 1=lid open, "
 		"-1=lid closed)");
 unsigned int i915_powersave __read_mostly = 1;
 module_param_named(powersave, i915_powersave, int, 0600);
 MODULE_PARM_DESC(powersave,
 		"Enable powersavings, fbc, downclocking, etc. (default: true)");
 int i915_semaphores __read_mostly = -1;
 module_param_named(semaphores, i915_semaphores, int, 0600);
 MODULE_PARM_DESC(semaphores,
 		"Use semaphores for inter-ring sync (default: -1 (use per-chip defaults))");
 int i915_enable_rc6 __read_mostly = -1;
 module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0600);
 MODULE_PARM_DESC(i915_enable_rc6,
 		"Enable power-saving render C-state 6 (default: -1 (use per-chip default)");
 int i915_enable_fbc __read_mostly = -1;
 module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600);
 MODULE_PARM_DESC(i915_enable_fbc,
 		"Enable frame buffer compression for power savings "
 		"(default: -1 (use per-chip default))");
 unsigned int i915_lvds_downclock __read_mostly = 0;
 module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
 MODULE_PARM_DESC(lvds_downclock,
 		"Use panel (LVDS/eDP) downclocking for power savings "
 		"(default: false)");
 int i915_panel_use_ssc __read_mostly = -1;
 module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
 MODULE_PARM_DESC(lvds_use_ssc,
 		"Use Spread Spectrum Clock with panels [LVDS/eDP] "
 		"(default: auto from VBT)");
 int i915_vbt_sdvo_panel_type __read_mostly = -1;
 module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600);
 MODULE_PARM_DESC(vbt_sdvo_panel_type,
 		"Override selection of SDVO panel mode in the VBT "
 		"(default: auto)");
 static bool i915_try_reset __read_mostly = true;
 module_param_named(reset, i915_try_reset, bool, 0600);
 MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
 bool i915_enable_hangcheck __read_mostly = true;
 module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644);
 MODULE_PARM_DESC(enable_hangcheck,
 		"Periodically check GPU activity for detecting hangs. "
 		"WARNING: Disabling this can cause system wide hangs. "
 		"(default: true)");
 static struct drm_driver driver;
 extern int intel_agp_enabled;
 #define INTEL_VGA_DEVICE(id, info) {		\
 	.class = PCI_BASE_CLASS_DISPLAY << 16,	\
 	.class_mask = 0xff0000,			\
 	.vendor = 0x8086,			\
 	.device = id,				\
 	.subvendor = PCI_ANY_ID,		\
 	.subdevice = PCI_ANY_ID,		\
 	.driver_data = (unsigned long) info }
 static const struct intel_device_info intel_i830_info = {
 	.gen = 2, .is_mobile = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_845g_info = {
 	.gen = 2,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i85x_info = {
 	.gen = 2, .is_i85x = 1, .is_mobile = 1,
 	.cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i865g_info = {
 	.gen = 2,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i915g_info = {
 	.gen = 3, .is_i915g = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i915gm_info = {
 	.gen = 3, .is_mobile = 1,
 	.cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 	.supports_tv = 1,
 };
 static const struct intel_device_info intel_i945g_info = {
 	.gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 };
 static const struct intel_device_info intel_i945gm_info = {
 	.gen = 3, .is_i945gm = 1, .is_mobile = 1,
 	.has_hotplug = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 	.supports_tv = 1,
 };
 static const struct intel_device_info intel_i965g_info = {
 	.gen = 4, .is_broadwater = 1,
 	.has_hotplug = 1,
 	.has_overlay = 1,
 };
 static const struct intel_device_info intel_i965gm_info = {
 	.gen = 4, .is_crestline = 1,
 	.is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
 	.has_overlay = 1,
 	.supports_tv = 1,
 };
 static const struct intel_device_info intel_g33_info = {
 	.gen = 3, .is_g33 = 1,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_overlay = 1,
 };
 static const struct intel_device_info intel_g45_info = {
 	.gen = 4, .is_g4x = 1, .need_gfx_hws = 1,
 	.has_pipe_cxsr = 1, .has_hotplug = 1,
 	.has_bsd_ring = 1,
 };
 static const struct intel_device_info intel_gm45_info = {
 	.gen = 4, .is_g4x = 1,
 	.is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
 	.has_pipe_cxsr = 1, .has_hotplug = 1,
 	.supports_tv = 1,
 	.has_bsd_ring = 1,
 };
 static const struct intel_device_info intel_pineview_info = {
 	.gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_overlay = 1,
 };
 static const struct intel_device_info intel_ironlake_d_info = {
 	.gen = 5,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_bsd_ring = 1,
 };
 static const struct intel_device_info intel_ironlake_m_info = {
 	.gen = 5, .is_mobile = 1,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_fbc = 1,
 	.has_bsd_ring = 1,
 };
 static const struct intel_device_info intel_sandybridge_d_info = {
 	.gen = 6,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
+	.has_llc = 1,
 };
 static const struct intel_device_info intel_sandybridge_m_info = {
 	.gen = 6, .is_mobile = 1,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_fbc = 1,
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
+	.has_llc = 1,
 };
 static const struct intel_device_info intel_ivybridge_d_info = {
 	.is_ivybridge = 1, .gen = 7,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
+	.has_llc = 1,
 };
 static const struct intel_device_info intel_ivybridge_m_info = {
 	.is_ivybridge = 1, .gen = 7, .is_mobile = 1,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_fbc = 0,	/* FBC is not enabled on Ivybridge mobile yet */
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
+	.has_llc = 1,
 };
 static const struct pci_device_id pciidlist[] = {		/* aka */
 	INTEL_VGA_DEVICE(0x3577, &intel_i830_info),		/* I830_M */
 	INTEL_VGA_DEVICE(0x2562, &intel_845g_info),		/* 845_G */
 	INTEL_VGA_DEVICE(0x3582, &intel_i85x_info),		/* I855_GM */
 	INTEL_VGA_DEVICE(0x358e, &intel_i85x_info),
 	INTEL_VGA_DEVICE(0x2572, &intel_i865g_info),		/* I865_G */
 	INTEL_VGA_DEVICE(0x2582, &intel_i915g_info),		/* I915_G */
 	INTEL_VGA_DEVICE(0x258a, &intel_i915g_info),		/* E7221_G */
 	INTEL_VGA_DEVICE(0x2592, &intel_i915gm_info),		/* I915_GM */
 	INTEL_VGA_DEVICE(0x2772, &intel_i945g_info),		/* I945_G */
 	INTEL_VGA_DEVICE(0x27a2, &intel_i945gm_info),		/* I945_GM */
 	INTEL_VGA_DEVICE(0x27ae, &intel_i945gm_info),		/* I945_GME */
 	INTEL_VGA_DEVICE(0x2972, &intel_i965g_info),		/* I946_GZ */
 	INTEL_VGA_DEVICE(0x2982, &intel_i965g_info),		/* G35_G */
 	INTEL_VGA_DEVICE(0x2992, &intel_i965g_info),		/* I965_Q */
 	INTEL_VGA_DEVICE(0x29a2, &intel_i965g_info),		/* I965_G */
 	INTEL_VGA_DEVICE(0x29b2, &intel_g33_info),		/* Q35_G */
 	INTEL_VGA_DEVICE(0x29c2, &intel_g33_info),		/* G33_G */
 	INTEL_VGA_DEVICE(0x29d2, &intel_g33_info),		/* Q33_G */
 	INTEL_VGA_DEVICE(0x2a02, &intel_i965gm_info),		/* I965_GM */
 	INTEL_VGA_DEVICE(0x2a12, &intel_i965gm_info),		/* I965_GME */
 	INTEL_VGA_DEVICE(0x2a42, &intel_gm45_info),		/* GM45_G */
 	INTEL_VGA_DEVICE(0x2e02, &intel_g45_info),		/* IGD_E_G */
 	INTEL_VGA_DEVICE(0x2e12, &intel_g45_info),		/* Q45_G */
 	INTEL_VGA_DEVICE(0x2e22, &intel_g45_info),		/* G45_G */
 	INTEL_VGA_DEVICE(0x2e32, &intel_g45_info),		/* G41_G */
 	INTEL_VGA_DEVICE(0x2e42, &intel_g45_info),		/* B43_G */
 	INTEL_VGA_DEVICE(0x2e92, &intel_g45_info),		/* B43_G.1 */
 	INTEL_VGA_DEVICE(0xa001, &intel_pineview_info),
 	INTEL_VGA_DEVICE(0xa011, &intel_pineview_info),
 	INTEL_VGA_DEVICE(0x0042, &intel_ironlake_d_info),
 	INTEL_VGA_DEVICE(0x0046, &intel_ironlake_m_info),
 	INTEL_VGA_DEVICE(0x0102, &intel_sandybridge_d_info),
 	INTEL_VGA_DEVICE(0x0112, &intel_sandybridge_d_info),
 	INTEL_VGA_DEVICE(0x0122, &intel_sandybridge_d_info),
 	INTEL_VGA_DEVICE(0x0106, &intel_sandybridge_m_info),
 	INTEL_VGA_DEVICE(0x0116, &intel_sandybridge_m_info),
 	INTEL_VGA_DEVICE(0x0126, &intel_sandybridge_m_info),
 	INTEL_VGA_DEVICE(0x010A, &intel_sandybridge_d_info),
 	INTEL_VGA_DEVICE(0x0156, &intel_ivybridge_m_info), /* GT1 mobile */
 	INTEL_VGA_DEVICE(0x0166, &intel_ivybridge_m_info), /* GT2 mobile */
 	INTEL_VGA_DEVICE(0x0152, &intel_ivybridge_d_info), /* GT1 desktop */
 	INTEL_VGA_DEVICE(0x0162, &intel_ivybridge_d_info), /* GT2 desktop */
 	INTEL_VGA_DEVICE(0x015a, &intel_ivybridge_d_info), /* GT1 server */
 	{0, 0, 0}
 };
 #if defined(CONFIG_DRM_I915_KMS)
 MODULE_DEVICE_TABLE(pci, pciidlist);
 #endif
 #define INTEL_PCH_DEVICE_ID_MASK	0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE	0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE	0x1c00
 #define INTEL_PCH_PPT_DEVICE_ID_TYPE	0x1e00
 void intel_detect_pch(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct pci_dev *pch;
 	/*
 	 * The reason to probe ISA bridge instead of Dev31:Fun0 is to
 	 * make graphics device passthrough work easy for VMM, that only
 	 * need to expose ISA bridge to let driver know the real hardware
 	 * underneath. This is a requirement from virtualization team.
 	 */
 	pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
 	if (pch) {
 		if (pch->vendor == PCI_VENDOR_ID_INTEL) {
 			int id;
 			id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
 			if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_IBX;
 				DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
 			} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found CougarPoint PCH\n");
 			} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 				/* PantherPoint is CPT compatible */
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found PatherPoint PCH\n");
 			}
 		}
 		pci_dev_put(pch);
 	}
 }
 void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
 	int count;
 	count = 0;
 	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
 		udelay(10);
 	I915_WRITE_NOTRACE(FORCEWAKE, 1);
 	POSTING_READ(FORCEWAKE);
 	count = 0;
 	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0)
 		udelay(10);
 }
 void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 {
 	int count;
 	count = 0;
 	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1))
 		udelay(10);
 	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1);
 	POSTING_READ(FORCEWAKE_MT);
 	count = 0;
 	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1) == 0)
 		udelay(10);
 }
 /*
  * Generally this is called implicitly by the register read function. However,
  * if some sequence requires the GT to not power down then this function should
  * be called at the beginning of the sequence followed by a call to
  * gen6_gt_force_wake_put() at the end of the sequence.
  */
 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
 	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
 	/* Forcewake is atomic in case we get in here without the lock */
 	if (atomic_add_return(1, &dev_priv->forcewake_count) == 1)
 		dev_priv->display.force_wake_get(dev_priv);
 }
 void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE_NOTRACE(FORCEWAKE, 0);
 	POSTING_READ(FORCEWAKE);
 }
 void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0);
 	POSTING_READ(FORCEWAKE_MT);
 }
 /*
  * see gen6_gt_force_wake_get()
  */
 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 {
 	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
 	if (atomic_dec_and_test(&dev_priv->forcewake_count))
 		dev_priv->display.force_wake_put(dev_priv);
 }
 void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 {
 	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 		int loop = 500;
 		u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
 		while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 			udelay(10);
 			fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
 		}
 		WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES);
 		dev_priv->gt_fifo_count = fifo;
 	}
 	dev_priv->gt_fifo_count--;
 }
 static int i915_drm_freeze(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	drm_kms_helper_poll_disable(dev);
 	pci_save_state(dev->pdev);
 	/* If KMS is active, we do the leavevt stuff here */
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		int error = i915_gem_idle(dev);
 		if (error) {
 			dev_err(&dev->pdev->dev,
 				"GEM idle failed, resume might fail\n");
 			return error;
 		}
 		drm_irq_uninstall(dev);
 	}
 	i915_save_state(dev);
 	intel_opregion_fini(dev);
 	/* Modeset on resume, not lid events */
 	dev_priv->modeset_on_lid = 0;
 	return 0;
 }
 int i915_suspend(struct drm_device *dev, pm_message_t state)
 {
 	int error;
 	if (!dev || !dev->dev_private) {
 		DRM_ERROR("dev: %p\n", dev);
 		DRM_ERROR("DRM not initialized, aborting suspend.\n");
 		return -ENODEV;
 	}
 	if (state.event == PM_EVENT_PRETHAW)
 		return 0;
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 	error = i915_drm_freeze(dev);
 	if (error)
 		return error;
 	if (state.event == PM_EVENT_SUSPEND) {
 		/* Shut down the device */
 		pci_disable_device(dev->pdev);
 		pci_set_power_state(dev->pdev, PCI_D3hot);
 	}
 	return 0;
 }
 static int i915_drm_thaw(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int error = 0;
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		mutex_lock(&dev->struct_mutex);
 		i915_gem_restore_gtt_mappings(dev);
 		mutex_unlock(&dev->struct_mutex);
 	}
 	i915_restore_state(dev);
 	intel_opregion_setup(dev);
 	/* KMS EnterVT equivalent */
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 		mutex_lock(&dev->struct_mutex);
 		dev_priv->mm.suspended = 0;
 		error = i915_gem_init_ringbuffer(dev);
 		mutex_unlock(&dev->struct_mutex);
 		if (HAS_PCH_SPLIT(dev))
 			ironlake_init_pch_refclk(dev);
 		drm_mode_config_reset(dev);
 		drm_irq_install(dev);
 		/* Resume the modeset for every activated CRTC */
 		drm_helper_resume_force_mode(dev);
 		if (IS_IRONLAKE_M(dev))
 			ironlake_enable_rc6(dev);
 	}
 	intel_opregion_init(dev);
 	dev_priv->modeset_on_lid = 0;
 	return error;
 }
 int i915_resume(struct drm_device *dev)
 {
 	int ret;
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 	if (pci_enable_device(dev->pdev))
 		return -EIO;
 	pci_set_master(dev->pdev);
 	ret = i915_drm_thaw(dev);
 	if (ret)
 		return ret;
 	drm_kms_helper_poll_enable(dev);
 	return 0;
 }
 static int i8xx_do_reset(struct drm_device *dev, u8 flags)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	if (IS_I85X(dev))
 		return -ENODEV;
 	I915_WRITE(D_STATE, I915_READ(D_STATE) | DSTATE_GFX_RESET_I830);
 	POSTING_READ(D_STATE);
 	if (IS_I830(dev) || IS_845G(dev)) {
 		I915_WRITE(DEBUG_RESET_I830,
 			   DEBUG_RESET_DISPLAY |
 			   DEBUG_RESET_RENDER |
 			   DEBUG_RESET_FULL);
 		POSTING_READ(DEBUG_RESET_I830);
 		msleep(1);
 		I915_WRITE(DEBUG_RESET_I830, 0);
 		POSTING_READ(DEBUG_RESET_I830);
 	}
 	msleep(1);
 	I915_WRITE(D_STATE, I915_READ(D_STATE) & ~DSTATE_GFX_RESET_I830);
 	POSTING_READ(D_STATE);
 	return 0;
 }
 static int i965_reset_complete(struct drm_device *dev)
 {
 	u8 gdrst;
 	pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
 	return gdrst & 0x1;
 }
 static int i965_do_reset(struct drm_device *dev, u8 flags)
 {
 	u8 gdrst;
 	/*
 	 * Set the domains we want to reset (GRDOM/bits 2 and 3) as
 	 * well as the reset bit (GR/bit 0).  Setting the GR bit
 	 * triggers the reset; when done, the hardware will clear it.
 	 */
 	pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
 	pci_write_config_byte(dev->pdev, I965_GDRST, gdrst | flags | 0x1);
 	return wait_for(i965_reset_complete(dev), 500);
 }
 static int ironlake_do_reset(struct drm_device *dev, u8 flags)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 	I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, gdrst | flags | 0x1);
 	return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 }
 static int gen6_do_reset(struct drm_device *dev, u8 flags)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL);
 	return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 }
 /**
  * i915_reset - reset chip after a hang
  * @dev: drm device to reset
  * @flags: reset domains
  *
  * Reset the chip.  Useful if a hang is detected. Returns zero on successful
  * reset or otherwise an error code.
  *
  * Procedure is fairly simple:
  *   - reset the chip using the reset reg
  *   - re-init context state
  *   - re-init hardware status page
  *   - re-init ring buffer
  *   - re-init interrupt state
  *   - re-init display
  */
 int i915_reset(struct drm_device *dev, u8 flags)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	/*
 	 * We really should only reset the display subsystem if we actually
 	 * need to
 	 */
 	bool need_display = true;
 	int ret;
 	if (!i915_try_reset)
 		return 0;
 	if (!mutex_trylock(&dev->struct_mutex))
 		return -EBUSY;
 	i915_gem_reset(dev);
 	ret = -ENODEV;
 	if (get_seconds() - dev_priv->last_gpu_reset < 5) {
 		DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
 	} else switch (INTEL_INFO(dev)->gen) {
 	case 7:
 	case 6:
 		ret = gen6_do_reset(dev, flags);
 		/* If reset with a user forcewake, try to restore */
 		if (atomic_read(&dev_priv->forcewake_count))
 			__gen6_gt_force_wake_get(dev_priv);
 		break;
 	case 5:
 		ret = ironlake_do_reset(dev, flags);
 		break;
 	case 4:
 		ret = i965_do_reset(dev, flags);
 		break;
 	case 2:
 		ret = i8xx_do_reset(dev, flags);
 		break;
 	}
 	dev_priv->last_gpu_reset = get_seconds();
 	if (ret) {
 		DRM_ERROR("Failed to reset chip.\n");
 		mutex_unlock(&dev->struct_mutex);
 		return ret;
 	}
 	/* Ok, now get things going again... */
 	/*
 	 * Everything depends on having the GTT running, so we need to start
 	 * there.  Fortunately we don't need to do this unless we reset the
 	 * chip at a PCI level.
 	 *
 	 * Next we need to restore the context, but we don't use those
 	 * yet either...
 	 *
 	 * Ring buffer needs to be re-initialized in the KMS case, or if X
 	 * was running at the time of the reset (i.e. we weren't VT
 	 * switched away).
 	 */
 	if (drm_core_check_feature(dev, DRIVER_MODESET) ||
 			!dev_priv->mm.suspended) {
 		dev_priv->mm.suspended = 0;
 		dev_priv->ring[RCS].init(&dev_priv->ring[RCS]);
 		if (HAS_BSD(dev))
 		    dev_priv->ring[VCS].init(&dev_priv->ring[VCS]);
 		if (HAS_BLT(dev))
 		    dev_priv->ring[BCS].init(&dev_priv->ring[BCS]);
 		mutex_unlock(&dev->struct_mutex);
 		drm_irq_uninstall(dev);
 		drm_mode_config_reset(dev);
 		drm_irq_install(dev);
 		mutex_lock(&dev->struct_mutex);
 	}
 	mutex_unlock(&dev->struct_mutex);
 	/*
 	 * Perform a full modeset as on later generations, e.g. Ironlake, we may
 	 * need to retrain the display link and cannot just restore the register
 	 * values.
 	 */
 	if (need_display) {
 		mutex_lock(&dev->mode_config.mutex);
 		drm_helper_resume_force_mode(dev);
 		mutex_unlock(&dev->mode_config.mutex);
 	}
 	return 0;
 }
 static int __devinit
 i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	/* Only bind to function 0 of the device. Early generations
 	 * used function 1 as a placeholder for multi-head. This causes
 	 * us confusion instead, especially on the systems where both
 	 * functions have the same PCI-ID!
 	 */
 	if (PCI_FUNC(pdev->devfn))
 		return -ENODEV;
 	return drm_get_pci_dev(pdev, ent, &driver);
 }
 static void
 i915_pci_remove(struct pci_dev *pdev)
 {
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	drm_put_dev(dev);
 }
 static int i915_pm_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
 	int error;
 	if (!drm_dev || !drm_dev->dev_private) {
 		dev_err(dev, "DRM not initialized, aborting suspend.\n");
 		return -ENODEV;
 	}
 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 	error = i915_drm_freeze(drm_dev);
 	if (error)
 		return error;
 	pci_disable_device(pdev);
 	pci_set_power_state(pdev, PCI_D3hot);
 	return 0;
 }
 static int i915_pm_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
 	return i915_resume(drm_dev);
 }
 static int i915_pm_freeze(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
 	if (!drm_dev || !drm_dev->dev_private) {
 		dev_err(dev, "DRM not initialized, aborting suspend.\n");
 		return -ENODEV;
 	}
 	return i915_drm_freeze(drm_dev);
 }
 static int i915_pm_thaw(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
 	return i915_drm_thaw(drm_dev);
 }
 static int i915_pm_poweroff(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
 	return i915_drm_freeze(drm_dev);
 }
 static const struct dev_pm_ops i915_pm_ops = {
 	.suspend = i915_pm_suspend,
 	.resume = i915_pm_resume,
 	.freeze = i915_pm_freeze,
 	.thaw = i915_pm_thaw,
 	.poweroff = i915_pm_poweroff,
 	.restore = i915_pm_resume,
 };
 static struct vm_operations_struct i915_gem_vm_ops = {
 	.fault = i915_gem_fault,
 	.open = drm_gem_vm_open,
 	.close = drm_gem_vm_close,
 };
 static const struct file_operations i915_driver_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
 	.fasync = drm_fasync,
 	.read = drm_read,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl = i915_compat_ioctl,
 #endif
 	.llseek = noop_llseek,
 };
 static struct drm_driver driver = {
 	/* Don't use MTRRs here; the Xserver or userspace app should
 	 * deal with them for Intel hardware.
 	 */
 	.driver_features =
 	    DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | /* DRIVER_USE_MTRR |*/
 	    DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM,
 	.load = i915_driver_load,
 	.unload = i915_driver_unload,
 	.open = i915_driver_open,
 	.lastclose = i915_driver_lastclose,
 	.preclose = i915_driver_preclose,
 	.postclose = i915_driver_postclose,
 	/* Used in place of i915_pm_ops for non-DRIVER_MODESET */
 	.suspend = i915_suspend,
 	.resume = i915_resume,
 	.device_is_agp = i915_driver_device_is_agp,
 	.reclaim_buffers = drm_core_reclaim_buffers,
 	.master_create = i915_master_create,
 	.master_destroy = i915_master_destroy,
 #if defined(CONFIG_DEBUG_FS)
 	.debugfs_init = i915_debugfs_init,
 	.debugfs_cleanup = i915_debugfs_cleanup,
 #endif
 	.gem_init_object = i915_gem_init_object,
 	.gem_free_object = i915_gem_free_object,
 	.gem_vm_ops = &i915_gem_vm_ops,
 	.dumb_create = i915_gem_dumb_create,
 	.dumb_map_offset = i915_gem_mmap_gtt,
 	.dumb_destroy = i915_gem_dumb_destroy,
 	.ioctls = i915_ioctls,
 	.fops = &i915_driver_fops,
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
 	.date = DRIVER_DATE,
 	.major = DRIVER_MAJOR,
 	.minor = DRIVER_MINOR,
 	.patchlevel = DRIVER_PATCHLEVEL,
 };
 static struct pci_driver i915_pci_driver = {
 	.name = DRIVER_NAME,
 	.id_table = pciidlist,
 	.probe = i915_pci_probe,
 	.remove = i915_pci_remove,
 	.driver.pm = &i915_pm_ops,
 };
 static int __init i915_init(void)
 {
 	if (!intel_agp_enabled) {
 		DRM_ERROR("drm/i915 can't work without intel_agp module!\n");
 		return -ENODEV;
 	}
 	driver.num_ioctls = i915_max_ioctl;
 	/*
 	 * If CONFIG_DRM_I915_KMS is set, default to KMS unless
 	 * explicitly disabled with the module pararmeter.
 	 *
 	 * Otherwise, just follow the parameter (defaulting to off).
 	 *
 	 * Allow optional vga_text_mode_force boot option to override
 	 * the default behavior.
 	 */
 #if defined(CONFIG_DRM_I915_KMS)
 	if (i915_modeset != 0)
 		driver.driver_features |= DRIVER_MODESET;
 #endif
 	if (i915_modeset == 1)
 		driver.driver_features |= DRIVER_MODESET;
 #ifdef CONFIG_VGA_CONSOLE
 	if (vgacon_text_force() && i915_modeset == -1)
 		driver.driver_features &= ~DRIVER_MODESET;
 #endif
 	if (!(driver.driver_features & DRIVER_MODESET))
 		driver.get_vblank_timestamp = NULL;
 	return drm_pci_init(&driver, &i915_pci_driver);
 }
 static void __exit i915_exit(void)
 {
 	drm_pci_exit(&driver, &i915_pci_driver);
 }
 module_init(i915_init);
 module_exit(i915_exit);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL and additional rights");
 #define __i915_read(x, y) \
 u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
 	u##x val = 0; \
 	if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 		gen6_gt_force_wake_get(dev_priv); \
 		val = read##y(dev_priv->regs + reg); \
 		gen6_gt_force_wake_put(dev_priv); \
 	} else { \
 		val = read##y(dev_priv->regs + reg); \
 	} \
 	trace_i915_reg_rw(false, reg, val, sizeof(val)); \
 	return val; \
 }
 __i915_read(8, b)
 __i915_read(16, w)
 __i915_read(32, l)
 __i915_read(64, q)
 #undef __i915_read
 #define __i915_write(x, y) \
 void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
 	trace_i915_reg_rw(true, reg, val, sizeof(val)); \
 	if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 		__gen6_gt_wait_for_fifo(dev_priv); \
 	} \
 	write##y(val, dev_priv->regs + reg); \
 }
 __i915_write(8, b)
 __i915_write(16, w)
 __i915_write(32, l)
 __i915_write(64, q)
 #undef __i915_write

drivers/gpu/drm/i915/i915_drv.h

Diff comments View file @ 3d29b84

 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*-
  */
 /*
  *
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * without limitation the rights to use, copy, modify, merge, publish,
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
 #ifndef _I915_DRV_H_
 #define _I915_DRV_H_
 #include "i915_reg.h"
 #include "intel_bios.h"
 #include "intel_ringbuffer.h"
 #include <linux/io-mapping.h>
 #include <linux/i2c.h>
 #include <drm/intel-gtt.h>
 #include <linux/backlight.h>
 /* General customization:
  */
 #define DRIVER_AUTHOR		"Tungsten Graphics, Inc."
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
 #define DRIVER_DATE		"20080730"
 enum pipe {
 	PIPE_A = 0,
 	PIPE_B,
 	PIPE_C,
 	I915_MAX_PIPES
 };
 #define pipe_name(p) ((p) + 'A')
 enum plane {
 	PLANE_A = 0,
 	PLANE_B,
 	PLANE_C,
 };
 #define plane_name(p) ((p) + 'A')
 #define I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
 #define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++)
 /* Interface history:
  *
  * 1.1: Original.
  * 1.2: Add Power Management
  * 1.3: Add vblank support
  * 1.4: Fix cmdbuffer path, add heap destroy
  * 1.5: Add vblank pipe configuration
  * 1.6: - New ioctl for scheduling buffer swaps on vertical blank
  *      - Support vertical blank on secondary display pipe
  */
 #define DRIVER_MAJOR		1
 #define DRIVER_MINOR		6
 #define DRIVER_PATCHLEVEL	0
 #define WATCH_COHERENCY	0
 #define WATCH_LISTS	0
 #define I915_GEM_PHYS_CURSOR_0 1
 #define I915_GEM_PHYS_CURSOR_1 2
 #define I915_GEM_PHYS_OVERLAY_REGS 3
 #define I915_MAX_PHYS_OBJECT (I915_GEM_PHYS_OVERLAY_REGS)
 struct drm_i915_gem_phys_object {
 	int id;
 	struct page **page_list;
 	drm_dma_handle_t *handle;
 	struct drm_i915_gem_object *cur_obj;
 };
 struct mem_block {
 	struct mem_block *next;
 	struct mem_block *prev;
 	int start;
 	int size;
 	struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */
 };
 struct opregion_header;
 struct opregion_acpi;
 struct opregion_swsci;
 struct opregion_asle;
 struct drm_i915_private;
 struct intel_opregion {
 	struct opregion_header *header;
 	struct opregion_acpi *acpi;
 	struct opregion_swsci *swsci;
 	struct opregion_asle *asle;
 	void *vbt;
 	u32 __iomem *lid_state;
 };
 #define OPREGION_SIZE            (8*1024)
 struct intel_overlay;
 struct intel_overlay_error_state;
 struct drm_i915_master_private {
 	drm_local_map_t *sarea;
 	struct _drm_i915_sarea *sarea_priv;
 };
 #define I915_FENCE_REG_NONE -1
 #define I915_MAX_NUM_FENCES 16
 /* 16 fences + sign bit for FENCE_REG_NONE */
 #define I915_MAX_NUM_FENCE_BITS 5
 struct drm_i915_fence_reg {
 	struct list_head lru_list;
 	struct drm_i915_gem_object *obj;
 	uint32_t setup_seqno;
 };
 struct sdvo_device_mapping {
 	u8 initialized;
 	u8 dvo_port;
 	u8 slave_addr;
 	u8 dvo_wiring;
 	u8 i2c_pin;
 	u8 ddc_pin;
 };
 struct intel_display_error_state;
 struct drm_i915_error_state {
 	u32 eir;
 	u32 pgtbl_er;
 	u32 pipestat[I915_MAX_PIPES];
 	u32 ipeir;
 	u32 ipehr;
 	u32 instdone;
 	u32 acthd;
 	u32 error; /* gen6+ */
 	u32 bcs_acthd; /* gen6+ blt engine */
 	u32 bcs_ipehr;
 	u32 bcs_ipeir;
 	u32 bcs_instdone;
 	u32 bcs_seqno;
 	u32 vcs_acthd; /* gen6+ bsd engine */
 	u32 vcs_ipehr;
 	u32 vcs_ipeir;
 	u32 vcs_instdone;
 	u32 vcs_seqno;
 	u32 instpm;
 	u32 instps;
 	u32 instdone1;
 	u32 seqno;
 	u64 bbaddr;
 	u64 fence[I915_MAX_NUM_FENCES];
 	struct timeval time;
 	struct drm_i915_error_object {
 		int page_count;
 		u32 gtt_offset;
 		u32 *pages[0];
 	} *ringbuffer[I915_NUM_RINGS], *batchbuffer[I915_NUM_RINGS];
 	struct drm_i915_error_buffer {
 		u32 size;
 		u32 name;
 		u32 seqno;
 		u32 gtt_offset;
 		u32 read_domains;
 		u32 write_domain;
 		s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
 		s32 pinned:2;
 		u32 tiling:2;
 		u32 dirty:1;
 		u32 purgeable:1;
 		u32 ring:4;
 		u32 cache_level:2;
 	} *active_bo, *pinned_bo;
 	u32 active_bo_count, pinned_bo_count;
 	struct intel_overlay_error_state *overlay;
 	struct intel_display_error_state *display;
 };
 struct drm_i915_display_funcs {
 	void (*dpms)(struct drm_crtc *crtc, int mode);
 	bool (*fbc_enabled)(struct drm_device *dev);
 	void (*enable_fbc)(struct drm_crtc *crtc, unsigned long interval);
 	void (*disable_fbc)(struct drm_device *dev);
 	int (*get_display_clock_speed)(struct drm_device *dev);
 	int (*get_fifo_size)(struct drm_device *dev, int plane);
 	void (*update_wm)(struct drm_device *dev);
 	void (*update_sprite_wm)(struct drm_device *dev, int pipe,
 				 uint32_t sprite_width, int pixel_size);
 	int (*crtc_mode_set)(struct drm_crtc *crtc,
 			     struct drm_display_mode *mode,
 			     struct drm_display_mode *adjusted_mode,
 			     int x, int y,
 			     struct drm_framebuffer *old_fb);
 	void (*write_eld)(struct drm_connector *connector,
 			  struct drm_crtc *crtc);
 	void (*fdi_link_train)(struct drm_crtc *crtc);
 	void (*init_clock_gating)(struct drm_device *dev);
 	void (*init_pch_clock_gating)(struct drm_device *dev);
 	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
 			  struct drm_framebuffer *fb,
 			  struct drm_i915_gem_object *obj);
 	int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 			    int x, int y);
 	void (*force_wake_get)(struct drm_i915_private *dev_priv);
 	void (*force_wake_put)(struct drm_i915_private *dev_priv);
 	/* clock updates for mode set */
 	/* cursor updates */
 	/* render clock increase/decrease */
 	/* display clock increase/decrease */
 	/* pll clock increase/decrease */
 };
 struct intel_device_info {
 	u8 gen;
 	u8 is_mobile:1;
 	u8 is_i85x:1;
 	u8 is_i915g:1;
 	u8 is_i945gm:1;
 	u8 is_g33:1;
 	u8 need_gfx_hws:1;
 	u8 is_g4x:1;
 	u8 is_pineview:1;
 	u8 is_broadwater:1;
 	u8 is_crestline:1;
 	u8 is_ivybridge:1;
 	u8 has_fbc:1;
 	u8 has_pipe_cxsr:1;
 	u8 has_hotplug:1;
 	u8 cursor_needs_physical:1;
 	u8 has_overlay:1;
 	u8 overlay_needs_physical:1;
 	u8 supports_tv:1;
 	u8 has_bsd_ring:1;
 	u8 has_blt_ring:1;
+	u8 has_llc:1;
 };
 enum no_fbc_reason {
 	FBC_NO_OUTPUT, /* no outputs enabled to compress */
 	FBC_STOLEN_TOO_SMALL, /* not enough space to hold compressed buffers */
 	FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */
 	FBC_MODE_TOO_LARGE, /* mode too large for compression */
 	FBC_BAD_PLANE, /* fbc not supported on plane */
 	FBC_NOT_TILED, /* buffer not tiled */
 	FBC_MULTIPLE_PIPES, /* more than one pipe active */
 	FBC_MODULE_PARAM,
 };
 enum intel_pch {
 	PCH_IBX,	/* Ibexpeak PCH */
 	PCH_CPT,	/* Cougarpoint PCH */
 };
 #define QUIRK_PIPEA_FORCE (1<<0)
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 struct intel_fbdev;
 struct intel_fbc_work;
 typedef struct drm_i915_private {
 	struct drm_device *dev;
 	const struct intel_device_info *info;
 	int has_gem;
 	int relative_constants_mode;
 	void __iomem *regs;
 	u32 gt_fifo_count;
 	struct intel_gmbus {
 		struct i2c_adapter adapter;
 		struct i2c_adapter *force_bit;
 		u32 reg0;
 	} *gmbus;
 	struct pci_dev *bridge_dev;
 	struct intel_ring_buffer ring[I915_NUM_RINGS];
 	uint32_t next_seqno;
 	drm_dma_handle_t *status_page_dmah;
 	uint32_t counter;
 	drm_local_map_t hws_map;
 	struct drm_i915_gem_object *pwrctx;
 	struct drm_i915_gem_object *renderctx;
 	struct resource mch_res;
 	unsigned int cpp;
 	int back_offset;
 	int front_offset;
 	int current_page;
 	int page_flipping;
 	atomic_t irq_received;
 	/* protects the irq masks */
 	spinlock_t irq_lock;
 	/** Cached value of IMR to avoid reads in updating the bitfield */
 	u32 pipestat[2];
 	u32 irq_mask;
 	u32 gt_irq_mask;
 	u32 pch_irq_mask;
 	u32 hotplug_supported_mask;
 	struct work_struct hotplug_work;
 	int tex_lru_log_granularity;
 	int allow_batchbuffer;
 	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
 	int vblank_pipe;
 	int num_pipe;
 	/* For hangcheck timer */
 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
 	struct timer_list hangcheck_timer;
 	int hangcheck_count;
 	uint32_t last_acthd;
 	uint32_t last_acthd_bsd;
 	uint32_t last_acthd_blt;
 	uint32_t last_instdone;
 	uint32_t last_instdone1;
 	unsigned long cfb_size;
 	unsigned int cfb_fb;
 	enum plane cfb_plane;
 	int cfb_y;
 	struct intel_fbc_work *fbc_work;
 	struct intel_opregion opregion;
 	/* overlay */
 	struct intel_overlay *overlay;
 	bool sprite_scaling_enabled;
 	/* LVDS info */
 	int backlight_level;  /* restore backlight to this value */
 	bool backlight_enabled;
 	struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
 	struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
 	/* Feature bits from the VBIOS */
 	unsigned int int_tv_support:1;
 	unsigned int lvds_dither:1;
 	unsigned int lvds_vbt:1;
 	unsigned int int_crt_support:1;
 	unsigned int lvds_use_ssc:1;
 	unsigned int display_clock_mode:1;
 	int lvds_ssc_freq;
 	struct {
 		int rate;
 		int lanes;
 		int preemphasis;
 		int vswing;
 		bool initialized;
 		bool support;
 		int bpp;
 		struct edp_power_seq pps;
 	} edp;
 	bool no_aux_handshake;
 	struct notifier_block lid_notifier;
 	int crt_ddc_pin;
 	struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
 	int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
 	int num_fence_regs; /* 8 on pre-965, 16 otherwise */
 	unsigned int fsb_freq, mem_freq, is_ddr3;
 	spinlock_t error_lock;
 	struct drm_i915_error_state *first_error;
 	struct work_struct error_work;
 	struct completion error_completion;
 	struct workqueue_struct *wq;
 	/* Display functions */
 	struct drm_i915_display_funcs display;
 	/* PCH chipset type */
 	enum intel_pch pch_type;
 	unsigned long quirks;
 	/* Register state */
 	bool modeset_on_lid;
 	u8 saveLBB;
 	u32 saveDSPACNTR;
 	u32 saveDSPBCNTR;
 	u32 saveDSPARB;
 	u32 saveHWS;
 	u32 savePIPEACONF;
 	u32 savePIPEBCONF;
 	u32 savePIPEASRC;
 	u32 savePIPEBSRC;
 	u32 saveFPA0;
 	u32 saveFPA1;
 	u32 saveDPLL_A;
 	u32 saveDPLL_A_MD;
 	u32 saveHTOTAL_A;
 	u32 saveHBLANK_A;
 	u32 saveHSYNC_A;
 	u32 saveVTOTAL_A;
 	u32 saveVBLANK_A;
 	u32 saveVSYNC_A;
 	u32 saveBCLRPAT_A;
 	u32 saveTRANSACONF;
 	u32 saveTRANS_HTOTAL_A;
 	u32 saveTRANS_HBLANK_A;
 	u32 saveTRANS_HSYNC_A;
 	u32 saveTRANS_VTOTAL_A;
 	u32 saveTRANS_VBLANK_A;
 	u32 saveTRANS_VSYNC_A;
 	u32 savePIPEASTAT;
 	u32 saveDSPASTRIDE;
 	u32 saveDSPASIZE;
 	u32 saveDSPAPOS;
 	u32 saveDSPAADDR;
 	u32 saveDSPASURF;
 	u32 saveDSPATILEOFF;
 	u32 savePFIT_PGM_RATIOS;
 	u32 saveBLC_HIST_CTL;
 	u32 saveBLC_PWM_CTL;
 	u32 saveBLC_PWM_CTL2;
 	u32 saveBLC_CPU_PWM_CTL;
 	u32 saveBLC_CPU_PWM_CTL2;
 	u32 saveFPB0;
 	u32 saveFPB1;
 	u32 saveDPLL_B;
 	u32 saveDPLL_B_MD;
 	u32 saveHTOTAL_B;
 	u32 saveHBLANK_B;
 	u32 saveHSYNC_B;
 	u32 saveVTOTAL_B;
 	u32 saveVBLANK_B;
 	u32 saveVSYNC_B;
 	u32 saveBCLRPAT_B;
 	u32 saveTRANSBCONF;
 	u32 saveTRANS_HTOTAL_B;
 	u32 saveTRANS_HBLANK_B;
 	u32 saveTRANS_HSYNC_B;
 	u32 saveTRANS_VTOTAL_B;
 	u32 saveTRANS_VBLANK_B;
 	u32 saveTRANS_VSYNC_B;
 	u32 savePIPEBSTAT;
 	u32 saveDSPBSTRIDE;
 	u32 saveDSPBSIZE;
 	u32 saveDSPBPOS;
 	u32 saveDSPBADDR;
 	u32 saveDSPBSURF;
 	u32 saveDSPBTILEOFF;
 	u32 saveVGA0;
 	u32 saveVGA1;
 	u32 saveVGA_PD;
 	u32 saveVGACNTRL;
 	u32 saveADPA;
 	u32 saveLVDS;
 	u32 savePP_ON_DELAYS;
 	u32 savePP_OFF_DELAYS;
 	u32 saveDVOA;
 	u32 saveDVOB;
 	u32 saveDVOC;
 	u32 savePP_ON;
 	u32 savePP_OFF;
 	u32 savePP_CONTROL;
 	u32 savePP_DIVISOR;
 	u32 savePFIT_CONTROL;
 	u32 save_palette_a[256];
 	u32 save_palette_b[256];
 	u32 saveDPFC_CB_BASE;
 	u32 saveFBC_CFB_BASE;
 	u32 saveFBC_LL_BASE;
 	u32 saveFBC_CONTROL;
 	u32 saveFBC_CONTROL2;
 	u32 saveIER;
 	u32 saveIIR;
 	u32 saveIMR;
 	u32 saveDEIER;
 	u32 saveDEIMR;
 	u32 saveGTIER;
 	u32 saveGTIMR;
 	u32 saveFDI_RXA_IMR;
 	u32 saveFDI_RXB_IMR;
 	u32 saveCACHE_MODE_0;
 	u32 saveMI_ARB_STATE;
 	u32 saveSWF0[16];
 	u32 saveSWF1[16];
 	u32 saveSWF2[3];
 	u8 saveMSR;
 	u8 saveSR[8];
 	u8 saveGR[25];
 	u8 saveAR_INDEX;
 	u8 saveAR[21];
 	u8 saveDACMASK;
 	u8 saveCR[37];
 	uint64_t saveFENCE[I915_MAX_NUM_FENCES];
 	u32 saveCURACNTR;
 	u32 saveCURAPOS;
 	u32 saveCURABASE;
 	u32 saveCURBCNTR;
 	u32 saveCURBPOS;
 	u32 saveCURBBASE;
 	u32 saveCURSIZE;
 	u32 saveDP_B;
 	u32 saveDP_C;
 	u32 saveDP_D;
 	u32 savePIPEA_GMCH_DATA_M;
 	u32 savePIPEB_GMCH_DATA_M;
 	u32 savePIPEA_GMCH_DATA_N;
 	u32 savePIPEB_GMCH_DATA_N;
 	u32 savePIPEA_DP_LINK_M;
 	u32 savePIPEB_DP_LINK_M;
 	u32 savePIPEA_DP_LINK_N;
 	u32 savePIPEB_DP_LINK_N;
 	u32 saveFDI_RXA_CTL;
 	u32 saveFDI_TXA_CTL;
 	u32 saveFDI_RXB_CTL;
 	u32 saveFDI_TXB_CTL;
 	u32 savePFA_CTL_1;
 	u32 savePFB_CTL_1;
 	u32 savePFA_WIN_SZ;
 	u32 savePFB_WIN_SZ;
 	u32 savePFA_WIN_POS;
 	u32 savePFB_WIN_POS;
 	u32 savePCH_DREF_CONTROL;
 	u32 saveDISP_ARB_CTL;
 	u32 savePIPEA_DATA_M1;
 	u32 savePIPEA_DATA_N1;
 	u32 savePIPEA_LINK_M1;
 	u32 savePIPEA_LINK_N1;
 	u32 savePIPEB_DATA_M1;
 	u32 savePIPEB_DATA_N1;
 	u32 savePIPEB_LINK_M1;
 	u32 savePIPEB_LINK_N1;
 	u32 saveMCHBAR_RENDER_STANDBY;
 	u32 savePCH_PORT_HOTPLUG;
 	struct {
 		/** Bridge to intel-gtt-ko */
 		const struct intel_gtt *gtt;
 		/** Memory allocator for GTT stolen memory */
 		struct drm_mm stolen;
 		/** Memory allocator for GTT */
 		struct drm_mm gtt_space;
 		/** List of all objects in gtt_space. Used to restore gtt
 		 * mappings on resume */
 		struct list_head gtt_list;
 		/** Usable portion of the GTT for GEM */
 		unsigned long gtt_start;
 		unsigned long gtt_mappable_end;
 		unsigned long gtt_end;
 		struct io_mapping *gtt_mapping;
 		int gtt_mtrr;
 		struct shrinker inactive_shrinker;
 		/**
 		 * List of objects currently involved in rendering.
 		 *
 		 * Includes buffers having the contents of their GPU caches
 		 * flushed, not necessarily primitives.  last_rendering_seqno
 		 * represents when the rendering involved will be completed.
 		 *
 		 * A reference is held on the buffer while on this list.
 		 */
 		struct list_head active_list;
 		/**
 		 * List of objects which are not in the ringbuffer but which
 		 * still have a write_domain which needs to be flushed before
 		 * unbinding.
 		 *
 		 * last_rendering_seqno is 0 while an object is in this list.
 		 *
 		 * A reference is held on the buffer while on this list.
 		 */
 		struct list_head flushing_list;
 		/**
 		 * LRU list of objects which are not in the ringbuffer and
 		 * are ready to unbind, but are still in the GTT.
 		 *
 		 * last_rendering_seqno is 0 while an object is in this list.
 		 *
 		 * A reference is not held on the buffer while on this list,
 		 * as merely being GTT-bound shouldn't prevent its being
 		 * freed, and we'll pull it off the list in the free path.
 		 */
 		struct list_head inactive_list;
 		/**
 		 * LRU list of objects which are not in the ringbuffer but
 		 * are still pinned in the GTT.
 		 */
 		struct list_head pinned_list;
 		/** LRU list of objects with fence regs on them. */
 		struct list_head fence_list;
 		/**
 		 * List of objects currently pending being freed.
 		 *
 		 * These objects are no longer in use, but due to a signal
 		 * we were prevented from freeing them at the appointed time.
 		 */
 		struct list_head deferred_free_list;
 		/**
 		 * We leave the user IRQ off as much as possible,
 		 * but this means that requests will finish and never
 		 * be retired once the system goes idle. Set a timer to
 		 * fire periodically while the ring is running. When it
 		 * fires, go retire requests.
 		 */
 		struct delayed_work retire_work;
 		/**
 		 * Are we in a non-interruptible section of code like
 		 * modesetting?
 		 */
 		bool interruptible;
 		/**
 		 * Flag if the X Server, and thus DRM, is not currently in
 		 * control of the device.
 		 *
 		 * This is set between LeaveVT and EnterVT.  It needs to be
 		 * replaced with a semaphore.  It also needs to be
 		 * transitioned away from for kernel modesetting.
 		 */
 		int suspended;
 		/**
 		 * Flag if the hardware appears to be wedged.
 		 *
 		 * This is set when attempts to idle the device timeout.
 		 * It prevents command submission from occurring and makes
 		 * every pending request fail
 		 */
 		atomic_t wedged;
 		/** Bit 6 swizzling required for X tiling */
 		uint32_t bit_6_swizzle_x;
 		/** Bit 6 swizzling required for Y tiling */
 		uint32_t bit_6_swizzle_y;
 		/* storage for physical objects */
 		struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
 		/* accounting, useful for userland debugging */
 		size_t gtt_total;
 		size_t mappable_gtt_total;
 		size_t object_memory;
 		u32 object_count;
 	} mm;
 	struct sdvo_device_mapping sdvo_mappings[2];
 	/* indicate whether the LVDS_BORDER should be enabled or not */
 	unsigned int lvds_border_bits;
 	/* Panel fitter placement and size for Ironlake+ */
 	u32 pch_pf_pos, pch_pf_size;
 	struct drm_crtc *plane_to_crtc_mapping[3];
 	struct drm_crtc *pipe_to_crtc_mapping[3];
 	wait_queue_head_t pending_flip_queue;
 	bool flip_pending_is_done;
 	/* Reclocking support */
 	bool render_reclock_avail;
 	bool lvds_downclock_avail;
 	/* indicates the reduced downclock for LVDS*/
 	int lvds_downclock;
 	struct work_struct idle_work;
 	struct timer_list idle_timer;
 	bool busy;
 	u16 orig_clock;
 	int child_dev_num;
 	struct child_device_config *child_dev;
 	struct drm_connector *int_lvds_connector;
 	struct drm_connector *int_edp_connector;
 	bool mchbar_need_disable;
 	struct work_struct rps_work;
 	spinlock_t rps_lock;
 	u32 pm_iir;
 	u8 cur_delay;
 	u8 min_delay;
 	u8 max_delay;
 	u8 fmax;
 	u8 fstart;
 	u64 last_count1;
 	unsigned long last_time1;
 	unsigned long chipset_power;
 	u64 last_count2;
 	struct timespec last_time2;
 	unsigned long gfx_power;
 	int c_m;
 	int r_t;
 	u8 corr;
 	spinlock_t *mchdev_lock;
 	enum no_fbc_reason no_fbc_reason;
 	struct drm_mm_node *compressed_fb;
 	struct drm_mm_node *compressed_llb;
 	unsigned long last_gpu_reset;
 	/* list of fbdev register on this device */
 	struct intel_fbdev *fbdev;
 	struct backlight_device *backlight;
 	struct drm_property *broadcast_rgb_property;
 	struct drm_property *force_audio_property;
 	atomic_t forcewake_count;
 } drm_i915_private_t;
 enum i915_cache_level {
 	I915_CACHE_NONE,
 	I915_CACHE_LLC,
 	I915_CACHE_LLC_MLC, /* gen6+ */
 };
 struct drm_i915_gem_object {
 	struct drm_gem_object base;
 	/** Current space allocated to this object in the GTT, if any. */
 	struct drm_mm_node *gtt_space;
 	struct list_head gtt_list;
 	/** This object's place on the active/flushing/inactive lists */
 	struct list_head ring_list;
 	struct list_head mm_list;
 	/** This object's place on GPU write list */
 	struct list_head gpu_write_list;
 	/** This object's place in the batchbuffer or on the eviction list */
 	struct list_head exec_list;
 	/**
 	 * This is set if the object is on the active or flushing lists
 	 * (has pending rendering), and is not set if it's on inactive (ready
 	 * to be unbound).
 	 */
 	unsigned int active:1;
 	/**
 	 * This is set if the object has been written to since last bound
 	 * to the GTT
 	 */
 	unsigned int dirty:1;
 	/**
 	 * This is set if the object has been written to since the last
 	 * GPU flush.
 	 */
 	unsigned int pending_gpu_write:1;
 	/**
 	 * Fence register bits (if any) for this object.  Will be set
 	 * as needed when mapped into the GTT.
 	 * Protected by dev->struct_mutex.
 	 */
 	signed int fence_reg:I915_MAX_NUM_FENCE_BITS;
 	/**
 	 * Advice: are the backing pages purgeable?
 	 */
 	unsigned int madv:2;
 	/**
 	 * Current tiling mode for the object.
 	 */
 	unsigned int tiling_mode:2;
 	unsigned int tiling_changed:1;
 	/** How many users have pinned this object in GTT space. The following
 	 * users can each hold at most one reference: pwrite/pread, pin_ioctl
 	 * (via user_pin_count), execbuffer (objects are not allowed multiple
 	 * times for the same batchbuffer), and the framebuffer code. When
 	 * switching/pageflipping, the framebuffer code has at most two buffers
 	 * pinned per crtc.
 	 *
 	 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
 	 * bits with absolutely no headroom. So use 4 bits. */
 	unsigned int pin_count:4;
 #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
 	/**
 	 * Is the object at the current location in the gtt mappable and
 	 * fenceable? Used to avoid costly recalculations.
 	 */
 	unsigned int map_and_fenceable:1;
 	/**
 	 * Whether the current gtt mapping needs to be mappable (and isn't just
 	 * mappable by accident). Track pin and fault separate for a more
 	 * accurate mappable working set.
 	 */
 	unsigned int fault_mappable:1;
 	unsigned int pin_mappable:1;
 	/*
 	 * Is the GPU currently using a fence to access this buffer,
 	 */
 	unsigned int pending_fenced_gpu_access:1;
 	unsigned int fenced_gpu_access:1;
 	unsigned int cache_level:2;
 	struct page **pages;
 	/**
 	 * DMAR support
 	 */
 	struct scatterlist *sg_list;
 	int num_sg;
 	/**
 	 * Used for performing relocations during execbuffer insertion.
 	 */
 	struct hlist_node exec_node;
 	unsigned long exec_handle;
 	struct drm_i915_gem_exec_object2 *exec_entry;
 	/**
 	 * Current offset of the object in GTT space.
 	 *
 	 * This is the same as gtt_space->start
 	 */
 	uint32_t gtt_offset;
 	/** Breadcrumb of last rendering to the buffer. */
 	uint32_t last_rendering_seqno;
 	struct intel_ring_buffer *ring;
 	/** Breadcrumb of last fenced GPU access to the buffer. */
 	uint32_t last_fenced_seqno;
 	struct intel_ring_buffer *last_fenced_ring;
 	/** Current tiling stride for the object, if it's tiled. */
 	uint32_t stride;
 	/** Record of address bit 17 of each page at last unbind. */
 	unsigned long *bit_17;
 	/**
 	 * If present, while GEM_DOMAIN_CPU is in the read domain this array
 	 * flags which individual pages are valid.
 	 */
 	uint8_t *page_cpu_valid;
 	/** User space pin count and filp owning the pin */
 	uint32_t user_pin_count;
 	struct drm_file *pin_filp;
 	/** for phy allocated objects */
 	struct drm_i915_gem_phys_object *phys_obj;
 	/**
 	 * Number of crtcs where this object is currently the fb, but
 	 * will be page flipped away on the next vblank.  When it
 	 * reaches 0, dev_priv->pending_flip_queue will be woken up.
 	 */
 	atomic_t pending_flip;
 };
 #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
 /**
  * Request queue structure.
  *
  * The request queue allows us to note sequence numbers that have been emitted
  * and may be associated with active buffers to be retired.
  *
  * By keeping this list, we can avoid having to do questionable
  * sequence-number comparisons on buffer last_rendering_seqnos, and associate
  * an emission time with seqnos for tracking how far ahead of the GPU we are.
  */
 struct drm_i915_gem_request {
 	/** On Which ring this request was generated */
 	struct intel_ring_buffer *ring;
 	/** GEM sequence number associated with this request. */
 	uint32_t seqno;
 	/** Time at which this request was emitted, in jiffies. */
 	unsigned long emitted_jiffies;
 	/** global list entry for this request */
 	struct list_head list;
 	struct drm_i915_file_private *file_priv;
 	/** file_priv list entry for this request */
 	struct list_head client_list;
 };
 struct drm_i915_file_private {
 	struct {
 		struct spinlock lock;
 		struct list_head request_list;
 	} mm;
 };
 #define INTEL_INFO(dev)	(((struct drm_i915_private *) (dev)->dev_private)->info)
 #define IS_I830(dev)		((dev)->pci_device == 0x3577)
 #define IS_845G(dev)		((dev)->pci_device == 0x2562)
 #define IS_I85X(dev)		(INTEL_INFO(dev)->is_i85x)
 #define IS_I865G(dev)		((dev)->pci_device == 0x2572)
 #define IS_I915G(dev)		(INTEL_INFO(dev)->is_i915g)
 #define IS_I915GM(dev)		((dev)->pci_device == 0x2592)
 #define IS_I945G(dev)		((dev)->pci_device == 0x2772)
 #define IS_I945GM(dev)		(INTEL_INFO(dev)->is_i945gm)
 #define IS_BROADWATER(dev)	(INTEL_INFO(dev)->is_broadwater)
 #define IS_CRESTLINE(dev)	(INTEL_INFO(dev)->is_crestline)
 #define IS_GM45(dev)		((dev)->pci_device == 0x2A42)
 #define IS_G4X(dev)		(INTEL_INFO(dev)->is_g4x)
 #define IS_PINEVIEW_G(dev)	((dev)->pci_device == 0xa001)
 #define IS_PINEVIEW_M(dev)	((dev)->pci_device == 0xa011)
 #define IS_PINEVIEW(dev)	(INTEL_INFO(dev)->is_pineview)
 #define IS_G33(dev)		(INTEL_INFO(dev)->is_g33)
 #define IS_IRONLAKE_D(dev)	((dev)->pci_device == 0x0042)
 #define IS_IRONLAKE_M(dev)	((dev)->pci_device == 0x0046)
 #define IS_IVYBRIDGE(dev)	(INTEL_INFO(dev)->is_ivybridge)
 #define IS_MOBILE(dev)		(INTEL_INFO(dev)->is_mobile)
 /*
  * The genX designation typically refers to the render engine, so render
  * capability related checks should use IS_GEN, while display and other checks
  * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
  * chips, etc.).
  */
 #define IS_GEN2(dev)	(INTEL_INFO(dev)->gen == 2)
 #define IS_GEN3(dev)	(INTEL_INFO(dev)->gen == 3)
 #define IS_GEN4(dev)	(INTEL_INFO(dev)->gen == 4)
 #define IS_GEN5(dev)	(INTEL_INFO(dev)->gen == 5)
 #define IS_GEN6(dev)	(INTEL_INFO(dev)->gen == 6)
 #define IS_GEN7(dev)	(INTEL_INFO(dev)->gen == 7)
 #define HAS_BSD(dev)            (INTEL_INFO(dev)->has_bsd_ring)
 #define HAS_BLT(dev)            (INTEL_INFO(dev)->has_blt_ring)
+#define HAS_LLC(dev)            (INTEL_INFO(dev)->has_llc)
 #define I915_NEED_GFX_HWS(dev)	(INTEL_INFO(dev)->need_gfx_hws)
 #define HAS_OVERLAY(dev)		(INTEL_INFO(dev)->has_overlay)
 #define OVERLAY_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->overlay_needs_physical)
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
  * rows, which changed the alignment requirements and fence programming.
  */
 #define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
 						      IS_I915GM(dev)))
 #define SUPPORTS_DIGITAL_OUTPUTS(dev)	(!IS_GEN2(dev) && !IS_PINEVIEW(dev))
 #define SUPPORTS_INTEGRATED_HDMI(dev)	(IS_G4X(dev) || IS_GEN5(dev))
 #define SUPPORTS_INTEGRATED_DP(dev)	(IS_G4X(dev) || IS_GEN5(dev))
 #define SUPPORTS_EDP(dev)		(IS_IRONLAKE_M(dev))
 #define SUPPORTS_TV(dev)		(INTEL_INFO(dev)->supports_tv)
 #define I915_HAS_HOTPLUG(dev)		 (INTEL_INFO(dev)->has_hotplug)
 /* dsparb controlled by hw only */
 #define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
 #define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
 #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
 #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
 #define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev))
 #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5)
 #define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
 #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
 #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
 #include "i915_trace.h"
 extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
 extern unsigned int i915_fbpercrtc __always_unused;
 extern int i915_panel_ignore_lid __read_mostly;
 extern unsigned int i915_powersave __read_mostly;
 extern int i915_semaphores __read_mostly;
 extern unsigned int i915_lvds_downclock __read_mostly;
 extern int i915_panel_use_ssc __read_mostly;
 extern int i915_vbt_sdvo_panel_type __read_mostly;
 extern int i915_enable_rc6 __read_mostly;
 extern int i915_enable_fbc __read_mostly;
 extern bool i915_enable_hangcheck __read_mostly;
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
 extern int i915_master_create(struct drm_device *dev, struct drm_master *master);
 extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master);
 				/* i915_dma.c */
 extern void i915_kernel_lost_context(struct drm_device * dev);
 extern int i915_driver_load(struct drm_device *, unsigned long flags);
 extern int i915_driver_unload(struct drm_device *);
 extern int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv);
 extern void i915_driver_lastclose(struct drm_device * dev);
 extern void i915_driver_preclose(struct drm_device *dev,
 				 struct drm_file *file_priv);
 extern void i915_driver_postclose(struct drm_device *dev,
 				  struct drm_file *file_priv);
 extern int i915_driver_device_is_agp(struct drm_device * dev);
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
 extern int i915_emit_box(struct drm_device *dev,
 			 struct drm_clip_rect *box,
 			 int DR1, int DR4);
 extern int i915_reset(struct drm_device *dev, u8 flags);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 /* i915_irq.c */
 void i915_hangcheck_elapsed(unsigned long data);
 void i915_handle_error(struct drm_device *dev, bool wedged);
 extern int i915_irq_emit(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 extern int i915_irq_wait(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 extern void intel_irq_init(struct drm_device *dev);
 extern int i915_vblank_pipe_set(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 extern int i915_vblank_pipe_get(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 extern int i915_vblank_swap(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 void
 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 void intel_enable_asle(struct drm_device *dev);
 #ifdef CONFIG_DEBUG_FS
 extern void i915_destroy_error_state(struct drm_device *dev);
 #else
 #define i915_destroy_error_state(x)
 #endif
 /* i915_gem.c */
 int i915_gem_init_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv);
 int i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv);
 int i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv);
 int i915_gem_execbuffer(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_execbuffer2(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv);
 int i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
 int i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
 int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
 int i915_gem_set_tiling(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_get_tiling(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
 int i915_gem_init_object(struct drm_gem_object *obj);
 int __must_check i915_gem_flush_ring(struct intel_ring_buffer *ring,
 				     uint32_t invalidate_domains,
 				     uint32_t flush_domains);
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
 						  size_t size);
 void i915_gem_free_object(struct drm_gem_object *obj);
 int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
 				     uint32_t alignment,
 				     bool map_and_fenceable);
 void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
 int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj);
 void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 				    struct intel_ring_buffer *ring,
 				    u32 seqno);
 int i915_gem_dumb_create(struct drm_file *file_priv,
 			 struct drm_device *dev,
 			 struct drm_mode_create_dumb *args);
 int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
 		      uint32_t handle, uint64_t *offset);
 int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev,
 			  uint32_t handle);
 /**
  * Returns true if seq1 is later than seq2.
  */
 static inline bool
 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
 {
 	return (int32_t)(seq1 - seq2) >= 0;
 }
 static inline u32
 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
 {
 	drm_i915_private_t *dev_priv = ring->dev->dev_private;
 	return ring->outstanding_lazy_request = dev_priv->next_seqno;
 }
 int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
 					   struct intel_ring_buffer *pipelined);
 int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
 void i915_gem_retire_requests(struct drm_device *dev);
 void i915_gem_reset(struct drm_device *dev);
 void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
 					    uint32_t read_domains,
 					    uint32_t write_domain);
 int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
 void i915_gem_do_init(struct drm_device *dev,
 		      unsigned long start,
 		      unsigned long mappable_end,
 		      unsigned long end);
 int __must_check i915_gpu_idle(struct drm_device *dev);
 int __must_check i915_gem_idle(struct drm_device *dev);
 int __must_check i915_add_request(struct intel_ring_buffer *ring,
 				  struct drm_file *file,
 				  struct drm_i915_gem_request *request);
 int __must_check i915_wait_request(struct intel_ring_buffer *ring,
 				   uint32_t seqno);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
 int __must_check
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
 				  bool write);
 int __must_check
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     u32 alignment,
 				     struct intel_ring_buffer *pipelined);
 int i915_gem_attach_phys_object(struct drm_device *dev,
 				struct drm_i915_gem_object *obj,
 				int id,
 				int align);
 void i915_gem_detach_phys_object(struct drm_device *dev,
 				 struct drm_i915_gem_object *obj);
 void i915_gem_free_all_phys_object(struct drm_device *dev);
 void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 uint32_t
 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 				    uint32_t size,
 				    int tiling_mode);
 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 				    enum i915_cache_level cache_level);
 /* i915_gem_gtt.c */
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
 void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
 				enum i915_cache_level cache_level);
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
 					  unsigned alignment, bool mappable);
 int __must_check i915_gem_evict_everything(struct drm_device *dev,
 					   bool purgeable_only);
 int __must_check i915_gem_evict_inactive(struct drm_device *dev,
 					 bool purgeable_only);
 /* i915_gem_tiling.c */
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
 void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
 void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 /* i915_gem_debug.c */
 void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
 			  const char *where, uint32_t mark);
 #if WATCH_LISTS
 int i915_verify_lists(struct drm_device *dev);
 #else
 #define i915_verify_lists(dev) 0
 #endif
 void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj,
 				     int handle);
 void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
 			  const char *where, uint32_t mark);
 /* i915_debugfs.c */
 int i915_debugfs_init(struct drm_minor *minor);
 void i915_debugfs_cleanup(struct drm_minor *minor);
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
 /* intel_i2c.c */
 extern int intel_setup_gmbus(struct drm_device *dev);
 extern void intel_teardown_gmbus(struct drm_device *dev);
 extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed);
 extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit);
 extern inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
 {
 	return container_of(adapter, struct intel_gmbus, adapter)->force_bit;
 }
 extern void intel_i2c_reset(struct drm_device *dev);
 /* intel_opregion.c */
 extern int intel_opregion_setup(struct drm_device *dev);
 #ifdef CONFIG_ACPI
 extern void intel_opregion_init(struct drm_device *dev);
 extern void intel_opregion_fini(struct drm_device *dev);
 extern void intel_opregion_asle_intr(struct drm_device *dev);
 extern void intel_opregion_gse_intr(struct drm_device *dev);
 extern void intel_opregion_enable_asle(struct drm_device *dev);
 #else
 static inline void intel_opregion_init(struct drm_device *dev) { return; }
 static inline void intel_opregion_fini(struct drm_device *dev) { return; }
 static inline void intel_opregion_asle_intr(struct drm_device *dev) { return; }
 static inline void intel_opregion_gse_intr(struct drm_device *dev) { return; }
 static inline void intel_opregion_enable_asle(struct drm_device *dev) { return; }
 #endif
 /* intel_acpi.c */
 #ifdef CONFIG_ACPI
 extern void intel_register_dsm_handler(void);
 extern void intel_unregister_dsm_handler(void);
 #else
 static inline void intel_register_dsm_handler(void) { return; }
 static inline void intel_unregister_dsm_handler(void) { return; }
 #endif /* CONFIG_ACPI */
 /* modesetting */
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
 extern bool intel_fbc_enabled(struct drm_device *dev);
 extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void ironlake_init_pch_refclk(struct drm_device *dev);
 extern void ironlake_enable_rc6(struct drm_device *dev);
 extern void gen6_set_rps(struct drm_device *dev, u8 val);
 extern void intel_detect_pch(struct drm_device *dev);
 extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
 extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv);
 /* overlay */
 #ifdef CONFIG_DEBUG_FS
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
 extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
 extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
 extern void intel_display_print_error_state(struct seq_file *m,
 					    struct drm_device *dev,
 					    struct intel_display_error_state *error);
 #endif
 #define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
 #define BEGIN_LP_RING(n) \
 	intel_ring_begin(LP_RING(dev_priv), (n))
 #define OUT_RING(x) \
 	intel_ring_emit(LP_RING(dev_priv), x)
 #define ADVANCE_LP_RING() \
 	intel_ring_advance(LP_RING(dev_priv))
 /**
  * Lock test for when it's just for synchronization of ring access.
  *
  * In that case, we don't need to do it when GEM is initialized as nobody else
  * has access to the ring.
  */
 #define RING_LOCK_TEST_WITH_RETURN(dev, file) do {			\
 	if (LP_RING(dev->dev_private)->obj == NULL)			\
 		LOCK_TEST_WITH_RETURN(dev, file);			\
 } while (0)
 /* On SNB platform, before reading ring registers forcewake bit
  * must be set to prevent GT core from power down and stale values being
  * returned.
  */
 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
 void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv);
 /* We give fast paths for the really cool registers */
 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 	(((dev_priv)->info->gen >= 6) && \
 	 ((reg) < 0x40000) &&		 \
 	 ((reg) != FORCEWAKE))
 #define __i915_read(x, y) \
 	u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg);
 __i915_read(8, b)
 __i915_read(16, w)
 __i915_read(32, l)
 __i915_read(64, q)
 #undef __i915_read
 #define __i915_write(x, y) \
 	void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val);
 __i915_write(8, b)
 __i915_write(16, w)
 __i915_write(32, l)
 __i915_write(64, q)
 #undef __i915_write
 #define I915_READ8(reg)		i915_read8(dev_priv, (reg))
 #define I915_WRITE8(reg, val)	i915_write8(dev_priv, (reg), (val))
 #define I915_READ16(reg)	i915_read16(dev_priv, (reg))
 #define I915_WRITE16(reg, val)	i915_write16(dev_priv, (reg), (val))
 #define I915_READ16_NOTRACE(reg)	readw(dev_priv->regs + (reg))
 #define I915_WRITE16_NOTRACE(reg, val)	writew(val, dev_priv->regs + (reg))
 #define I915_READ(reg)		i915_read32(dev_priv, (reg))
 #define I915_WRITE(reg, val)	i915_write32(dev_priv, (reg), (val))
 #define I915_READ_NOTRACE(reg)		readl(dev_priv->regs + (reg))
 #define I915_WRITE_NOTRACE(reg, val)	writel(val, dev_priv->regs + (reg))
 #define I915_WRITE64(reg, val)	i915_write64(dev_priv, (reg), (val))
 #define I915_READ64(reg)	i915_read64(dev_priv, (reg))
 #define POSTING_READ(reg)	(void)I915_READ_NOTRACE(reg)
 #define POSTING_READ16(reg)	(void)I915_READ16_NOTRACE(reg)
 #endif

drivers/gpu/drm/i915/i915_gem.c

Diff comments View file @ 3d29b84

1	/*	1	/*
2	* Copyright © 2008 Intel Corporation	2	* Copyright © 2008 Intel Corporation
3	*	3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a	4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),	5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation	6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the	8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:	9	* Software is furnished to do so, subject to the following conditions:
10	*	10	*
11	* The above copyright notice and this permission notice (including the next	11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the	12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.	13	* Software.
14	*	14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.	21	* IN THE SOFTWARE.
22	*	22	*
23	* Authors:	23	* Authors:
24	* Eric Anholt <eric@anholt.net>	24	* Eric Anholt <eric@anholt.net>
25	*	25	*
26	*/	26	*/
27		27
28	#include "drmP.h"	28	#include "drmP.h"
29	#include "drm.h"	29	#include "drm.h"
30	#include "i915_drm.h"	30	#include "i915_drm.h"
31	#include "i915_drv.h"	31	#include "i915_drv.h"
32	#include "i915_trace.h"	32	#include "i915_trace.h"
33	#include "intel_drv.h"	33	#include "intel_drv.h"
34	#include <linux/shmem_fs.h>	34	#include <linux/shmem_fs.h>
35	#include <linux/slab.h>	35	#include <linux/slab.h>
36	#include <linux/swap.h>	36	#include <linux/swap.h>
37	#include <linux/pci.h>	37	#include <linux/pci.h>
38		38
39	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);	39	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);	40	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);	41	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,	42	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43	bool write);	43	bool write);
44	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,	44	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45	uint64_t offset,	45	uint64_t offset,
46	uint64_t size);	46	uint64_t size);
47	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);	47	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
48	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,	48	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
49	unsigned alignment,	49	unsigned alignment,
50	bool map_and_fenceable);	50	bool map_and_fenceable);
51	static void i915_gem_clear_fence_reg(struct drm_device *dev,	51	static void i915_gem_clear_fence_reg(struct drm_device *dev,
52	struct drm_i915_fence_reg *reg);	52	struct drm_i915_fence_reg *reg);
53	static int i915_gem_phys_pwrite(struct drm_device *dev,	53	static int i915_gem_phys_pwrite(struct drm_device *dev,
54	struct drm_i915_gem_object *obj,	54	struct drm_i915_gem_object *obj,
55	struct drm_i915_gem_pwrite *args,	55	struct drm_i915_gem_pwrite *args,
56	struct drm_file *file);	56	struct drm_file *file);
57	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);	57	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
58		58
59	static int i915_gem_inactive_shrink(struct shrinker *shrinker,	59	static int i915_gem_inactive_shrink(struct shrinker *shrinker,
60	struct shrink_control *sc);	60	struct shrink_control *sc);
61		61
62	/* some bookkeeping */	62	/* some bookkeeping */
63	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,	63	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
64	size_t size)	64	size_t size)
65	{	65	{
66	dev_priv->mm.object_count++;	66	dev_priv->mm.object_count++;
67	dev_priv->mm.object_memory += size;	67	dev_priv->mm.object_memory += size;
68	}	68	}
69		69
70	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,	70	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
71	size_t size)	71	size_t size)
72	{	72	{
73	dev_priv->mm.object_count--;	73	dev_priv->mm.object_count--;
74	dev_priv->mm.object_memory -= size;	74	dev_priv->mm.object_memory -= size;
75	}	75	}
76		76
77	static int	77	static int
78	i915_gem_wait_for_error(struct drm_device *dev)	78	i915_gem_wait_for_error(struct drm_device *dev)
79	{	79	{
80	struct drm_i915_private *dev_priv = dev->dev_private;	80	struct drm_i915_private *dev_priv = dev->dev_private;
81	struct completion *x = &dev_priv->error_completion;	81	struct completion *x = &dev_priv->error_completion;
82	unsigned long flags;	82	unsigned long flags;
83	int ret;	83	int ret;
84		84
85	if (!atomic_read(&dev_priv->mm.wedged))	85	if (!atomic_read(&dev_priv->mm.wedged))
86	return 0;	86	return 0;
87		87
88	ret = wait_for_completion_interruptible(x);	88	ret = wait_for_completion_interruptible(x);
89	if (ret)	89	if (ret)
90	return ret;	90	return ret;
91		91
92	if (atomic_read(&dev_priv->mm.wedged)) {	92	if (atomic_read(&dev_priv->mm.wedged)) {
93	/* GPU is hung, bump the completion count to account for	93	/* GPU is hung, bump the completion count to account for
94	* the token we just consumed so that we never hit zero and	94	* the token we just consumed so that we never hit zero and
95	* end up waiting upon a subsequent completion event that	95	* end up waiting upon a subsequent completion event that
96	* will never happen.	96	* will never happen.
97	*/	97	*/
98	spin_lock_irqsave(&x->wait.lock, flags);	98	spin_lock_irqsave(&x->wait.lock, flags);
99	x->done++;	99	x->done++;
100	spin_unlock_irqrestore(&x->wait.lock, flags);	100	spin_unlock_irqrestore(&x->wait.lock, flags);
101	}	101	}
102	return 0;	102	return 0;
103	}	103	}
104		104
105	int i915_mutex_lock_interruptible(struct drm_device *dev)	105	int i915_mutex_lock_interruptible(struct drm_device *dev)
106	{	106	{
107	int ret;	107	int ret;
108		108
109	ret = i915_gem_wait_for_error(dev);	109	ret = i915_gem_wait_for_error(dev);
110	if (ret)	110	if (ret)
111	return ret;	111	return ret;
112		112
113	ret = mutex_lock_interruptible(&dev->struct_mutex);	113	ret = mutex_lock_interruptible(&dev->struct_mutex);
114	if (ret)	114	if (ret)
115	return ret;	115	return ret;
116		116
117	WARN_ON(i915_verify_lists(dev));	117	WARN_ON(i915_verify_lists(dev));
118	return 0;	118	return 0;
119	}	119	}
120		120
121	static inline bool	121	static inline bool
122	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)	122	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
123	{	123	{
124	return obj->gtt_space && !obj->active && obj->pin_count == 0;	124	return obj->gtt_space && !obj->active && obj->pin_count == 0;
125	}	125	}
126		126
127	void i915_gem_do_init(struct drm_device *dev,	127	void i915_gem_do_init(struct drm_device *dev,
128	unsigned long start,	128	unsigned long start,
129	unsigned long mappable_end,	129	unsigned long mappable_end,
130	unsigned long end)	130	unsigned long end)
131	{	131	{
132	drm_i915_private_t *dev_priv = dev->dev_private;	132	drm_i915_private_t *dev_priv = dev->dev_private;
133		133
134	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);	134	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
135		135
136	dev_priv->mm.gtt_start = start;	136	dev_priv->mm.gtt_start = start;
137	dev_priv->mm.gtt_mappable_end = mappable_end;	137	dev_priv->mm.gtt_mappable_end = mappable_end;
138	dev_priv->mm.gtt_end = end;	138	dev_priv->mm.gtt_end = end;
139	dev_priv->mm.gtt_total = end - start;	139	dev_priv->mm.gtt_total = end - start;
140	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;	140	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
141		141
142	/* Take over this portion of the GTT */	142	/* Take over this portion of the GTT */
143	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);	143	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
144	}	144	}
145		145
146	int	146	int
147	i915_gem_init_ioctl(struct drm_device dev, void data,	147	i915_gem_init_ioctl(struct drm_device dev, void data,
148	struct drm_file *file)	148	struct drm_file *file)
149	{	149	{
150	struct drm_i915_gem_init *args = data;	150	struct drm_i915_gem_init *args = data;
151		151
152	if (args->gtt_start >= args->gtt_end \|\|	152	if (args->gtt_start >= args->gtt_end \|\|
153	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))	153	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
154	return -EINVAL;	154	return -EINVAL;
155		155
156	mutex_lock(&dev->struct_mutex);	156	mutex_lock(&dev->struct_mutex);
157	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);	157	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
158	mutex_unlock(&dev->struct_mutex);	158	mutex_unlock(&dev->struct_mutex);
159		159
160	return 0;	160	return 0;
161	}	161	}
162		162
163	int	163	int
164	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,	164	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
165	struct drm_file *file)	165	struct drm_file *file)
166	{	166	{
167	struct drm_i915_private *dev_priv = dev->dev_private;	167	struct drm_i915_private *dev_priv = dev->dev_private;
168	struct drm_i915_gem_get_aperture *args = data;	168	struct drm_i915_gem_get_aperture *args = data;
169	struct drm_i915_gem_object *obj;	169	struct drm_i915_gem_object *obj;
170	size_t pinned;	170	size_t pinned;
171		171
172	if (!(dev->driver->driver_features & DRIVER_GEM))	172	if (!(dev->driver->driver_features & DRIVER_GEM))
173	return -ENODEV;	173	return -ENODEV;
174		174
175	pinned = 0;	175	pinned = 0;
176	mutex_lock(&dev->struct_mutex);	176	mutex_lock(&dev->struct_mutex);
177	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)	177	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
178	pinned += obj->gtt_space->size;	178	pinned += obj->gtt_space->size;
179	mutex_unlock(&dev->struct_mutex);	179	mutex_unlock(&dev->struct_mutex);
180		180
181	args->aper_size = dev_priv->mm.gtt_total;	181	args->aper_size = dev_priv->mm.gtt_total;
182	args->aper_available_size = args->aper_size - pinned;	182	args->aper_available_size = args->aper_size - pinned;
183		183
184	return 0;	184	return 0;
185	}	185	}
186		186
187	static int	187	static int
188	i915_gem_create(struct drm_file *file,	188	i915_gem_create(struct drm_file *file,
189	struct drm_device *dev,	189	struct drm_device *dev,
190	uint64_t size,	190	uint64_t size,
191	uint32_t *handle_p)	191	uint32_t *handle_p)
192	{	192	{
193	struct drm_i915_gem_object *obj;	193	struct drm_i915_gem_object *obj;
194	int ret;	194	int ret;
195	u32 handle;	195	u32 handle;
196		196
197	size = roundup(size, PAGE_SIZE);	197	size = roundup(size, PAGE_SIZE);
198	if (size == 0)	198	if (size == 0)
199	return -EINVAL;	199	return -EINVAL;
200		200
201	/* Allocate the new object */	201	/* Allocate the new object */
202	obj = i915_gem_alloc_object(dev, size);	202	obj = i915_gem_alloc_object(dev, size);
203	if (obj == NULL)	203	if (obj == NULL)
204	return -ENOMEM;	204	return -ENOMEM;
205		205
206	ret = drm_gem_handle_create(file, &obj->base, &handle);	206	ret = drm_gem_handle_create(file, &obj->base, &handle);
207	if (ret) {	207	if (ret) {
208	drm_gem_object_release(&obj->base);	208	drm_gem_object_release(&obj->base);
209	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);	209	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
210	kfree(obj);	210	kfree(obj);
211	return ret;	211	return ret;
212	}	212	}
213		213
214	/* drop reference from allocate - handle holds it now */	214	/* drop reference from allocate - handle holds it now */
215	drm_gem_object_unreference(&obj->base);	215	drm_gem_object_unreference(&obj->base);
216	trace_i915_gem_object_create(obj);	216	trace_i915_gem_object_create(obj);
217		217
218	*handle_p = handle;	218	*handle_p = handle;
219	return 0;	219	return 0;
220	}	220	}
221		221
222	int	222	int
223	i915_gem_dumb_create(struct drm_file *file,	223	i915_gem_dumb_create(struct drm_file *file,
224	struct drm_device *dev,	224	struct drm_device *dev,
225	struct drm_mode_create_dumb *args)	225	struct drm_mode_create_dumb *args)
226	{	226	{
227	/* have to work out size/pitch and return them */	227	/* have to work out size/pitch and return them */
228	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);	228	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
229	args->size = args->pitch * args->height;	229	args->size = args->pitch * args->height;
230	return i915_gem_create(file, dev,	230	return i915_gem_create(file, dev,
231	args->size, &args->handle);	231	args->size, &args->handle);
232	}	232	}
233		233
234	int i915_gem_dumb_destroy(struct drm_file *file,	234	int i915_gem_dumb_destroy(struct drm_file *file,
235	struct drm_device *dev,	235	struct drm_device *dev,
236	uint32_t handle)	236	uint32_t handle)
237	{	237	{
238	return drm_gem_handle_delete(file, handle);	238	return drm_gem_handle_delete(file, handle);
239	}	239	}
240		240
241	/**	241	/**
242	* Creates a new mm object and returns a handle to it.	242	* Creates a new mm object and returns a handle to it.
243	*/	243	*/
244	int	244	int
245	i915_gem_create_ioctl(struct drm_device dev, void data,	245	i915_gem_create_ioctl(struct drm_device dev, void data,
246	struct drm_file *file)	246	struct drm_file *file)
247	{	247	{
248	struct drm_i915_gem_create *args = data;	248	struct drm_i915_gem_create *args = data;
249	return i915_gem_create(file, dev,	249	return i915_gem_create(file, dev,
250	args->size, &args->handle);	250	args->size, &args->handle);
251	}	251	}
252		252
253	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)	253	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
254	{	254	{
255	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;	255	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
256		256
257	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&	257	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
258	obj->tiling_mode != I915_TILING_NONE;	258	obj->tiling_mode != I915_TILING_NONE;
259	}	259	}
260		260
261	static inline void	261	static inline void
262	slow_shmem_copy(struct page *dst_page,	262	slow_shmem_copy(struct page *dst_page,
263	int dst_offset,	263	int dst_offset,
264	struct page *src_page,	264	struct page *src_page,
265	int src_offset,	265	int src_offset,
266	int length)	266	int length)
267	{	267	{
268	char dst_vaddr, src_vaddr;	268	char dst_vaddr, src_vaddr;
269		269
270	dst_vaddr = kmap(dst_page);	270	dst_vaddr = kmap(dst_page);
271	src_vaddr = kmap(src_page);	271	src_vaddr = kmap(src_page);
272		272
273	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);	273	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
274		274
275	kunmap(src_page);	275	kunmap(src_page);
276	kunmap(dst_page);	276	kunmap(dst_page);
277	}	277	}
278		278
279	static inline void	279	static inline void
280	slow_shmem_bit17_copy(struct page *gpu_page,	280	slow_shmem_bit17_copy(struct page *gpu_page,
281	int gpu_offset,	281	int gpu_offset,
282	struct page *cpu_page,	282	struct page *cpu_page,
283	int cpu_offset,	283	int cpu_offset,
284	int length,	284	int length,
285	int is_read)	285	int is_read)
286	{	286	{
287	char gpu_vaddr, cpu_vaddr;	287	char gpu_vaddr, cpu_vaddr;
288		288
289	/* Use the unswizzled path if this page isn't affected. */	289	/* Use the unswizzled path if this page isn't affected. */
290	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {	290	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
291	if (is_read)	291	if (is_read)
292	return slow_shmem_copy(cpu_page, cpu_offset,	292	return slow_shmem_copy(cpu_page, cpu_offset,
293	gpu_page, gpu_offset, length);	293	gpu_page, gpu_offset, length);
294	else	294	else
295	return slow_shmem_copy(gpu_page, gpu_offset,	295	return slow_shmem_copy(gpu_page, gpu_offset,
296	cpu_page, cpu_offset, length);	296	cpu_page, cpu_offset, length);
297	}	297	}
298		298
299	gpu_vaddr = kmap(gpu_page);	299	gpu_vaddr = kmap(gpu_page);
300	cpu_vaddr = kmap(cpu_page);	300	cpu_vaddr = kmap(cpu_page);
301		301
302	/* Copy the data, XORing A6 with A17 (1). The user already knows he's	302	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
303	* XORing with the other bits (A9 for Y, A9 and A10 for X)	303	* XORing with the other bits (A9 for Y, A9 and A10 for X)
304	*/	304	*/
305	while (length > 0) {	305	while (length > 0) {
306	int cacheline_end = ALIGN(gpu_offset + 1, 64);	306	int cacheline_end = ALIGN(gpu_offset + 1, 64);
307	int this_length = min(cacheline_end - gpu_offset, length);	307	int this_length = min(cacheline_end - gpu_offset, length);
308	int swizzled_gpu_offset = gpu_offset ^ 64;	308	int swizzled_gpu_offset = gpu_offset ^ 64;
309		309
310	if (is_read) {	310	if (is_read) {
311	memcpy(cpu_vaddr + cpu_offset,	311	memcpy(cpu_vaddr + cpu_offset,
312	gpu_vaddr + swizzled_gpu_offset,	312	gpu_vaddr + swizzled_gpu_offset,
313	this_length);	313	this_length);
314	} else {	314	} else {
315	memcpy(gpu_vaddr + swizzled_gpu_offset,	315	memcpy(gpu_vaddr + swizzled_gpu_offset,
316	cpu_vaddr + cpu_offset,	316	cpu_vaddr + cpu_offset,
317	this_length);	317	this_length);
318	}	318	}
319	cpu_offset += this_length;	319	cpu_offset += this_length;
320	gpu_offset += this_length;	320	gpu_offset += this_length;
321	length -= this_length;	321	length -= this_length;
322	}	322	}
323		323
324	kunmap(cpu_page);	324	kunmap(cpu_page);
325	kunmap(gpu_page);	325	kunmap(gpu_page);
326	}	326	}
327		327
328	/**	328	/**
329	* This is the fast shmem pread path, which attempts to copy_from_user directly	329	* This is the fast shmem pread path, which attempts to copy_from_user directly
330	* from the backing pages of the object to the user's address space. On a	330	* from the backing pages of the object to the user's address space. On a
331	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().	331	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
332	*/	332	*/
333	static int	333	static int
334	i915_gem_shmem_pread_fast(struct drm_device *dev,	334	i915_gem_shmem_pread_fast(struct drm_device *dev,
335	struct drm_i915_gem_object *obj,	335	struct drm_i915_gem_object *obj,
336	struct drm_i915_gem_pread *args,	336	struct drm_i915_gem_pread *args,
337	struct drm_file *file)	337	struct drm_file *file)
338	{	338	{
339	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	339	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
340	ssize_t remain;	340	ssize_t remain;
341	loff_t offset;	341	loff_t offset;
342	char __user *user_data;	342	char __user *user_data;
343	int page_offset, page_length;	343	int page_offset, page_length;
344		344
345	user_data = (char __user *) (uintptr_t) args->data_ptr;	345	user_data = (char __user *) (uintptr_t) args->data_ptr;
346	remain = args->size;	346	remain = args->size;
347		347
348	offset = args->offset;	348	offset = args->offset;
349		349
350	while (remain > 0) {	350	while (remain > 0) {
351	struct page *page;	351	struct page *page;
352	char *vaddr;	352	char *vaddr;
353	int ret;	353	int ret;
354		354
355	/* Operation in this page	355	/* Operation in this page
356	*	356	*
357	* page_offset = offset within page	357	* page_offset = offset within page
358	* page_length = bytes to copy for this page	358	* page_length = bytes to copy for this page
359	*/	359	*/
360	page_offset = offset_in_page(offset);	360	page_offset = offset_in_page(offset);
361	page_length = remain;	361	page_length = remain;
362	if ((page_offset + remain) > PAGE_SIZE)	362	if ((page_offset + remain) > PAGE_SIZE)
363	page_length = PAGE_SIZE - page_offset;	363	page_length = PAGE_SIZE - page_offset;
364		364
365	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);	365	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
366	if (IS_ERR(page))	366	if (IS_ERR(page))
367	return PTR_ERR(page);	367	return PTR_ERR(page);
368		368
369	vaddr = kmap_atomic(page);	369	vaddr = kmap_atomic(page);
370	ret = __copy_to_user_inatomic(user_data,	370	ret = __copy_to_user_inatomic(user_data,
371	vaddr + page_offset,	371	vaddr + page_offset,
372	page_length);	372	page_length);
373	kunmap_atomic(vaddr);	373	kunmap_atomic(vaddr);
374		374
375	mark_page_accessed(page);	375	mark_page_accessed(page);
376	page_cache_release(page);	376	page_cache_release(page);
377	if (ret)	377	if (ret)
378	return -EFAULT;	378	return -EFAULT;
379		379
380	remain -= page_length;	380	remain -= page_length;
381	user_data += page_length;	381	user_data += page_length;
382	offset += page_length;	382	offset += page_length;
383	}	383	}
384		384
385	return 0;	385	return 0;
386	}	386	}
387		387
388	/**	388	/**
389	* This is the fallback shmem pread path, which allocates temporary storage	389	* This is the fallback shmem pread path, which allocates temporary storage
390	* in kernel space to copy_to_user into outside of the struct_mutex, so we	390	* in kernel space to copy_to_user into outside of the struct_mutex, so we
391	* can copy out of the object's backing pages while holding the struct mutex	391	* can copy out of the object's backing pages while holding the struct mutex
392	* and not take page faults.	392	* and not take page faults.
393	*/	393	*/
394	static int	394	static int
395	i915_gem_shmem_pread_slow(struct drm_device *dev,	395	i915_gem_shmem_pread_slow(struct drm_device *dev,
396	struct drm_i915_gem_object *obj,	396	struct drm_i915_gem_object *obj,
397	struct drm_i915_gem_pread *args,	397	struct drm_i915_gem_pread *args,
398	struct drm_file *file)	398	struct drm_file *file)
399	{	399	{
400	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	400	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
401	struct mm_struct *mm = current->mm;	401	struct mm_struct *mm = current->mm;
402	struct page **user_pages;	402	struct page **user_pages;
403	ssize_t remain;	403	ssize_t remain;
404	loff_t offset, pinned_pages, i;	404	loff_t offset, pinned_pages, i;
405	loff_t first_data_page, last_data_page, num_pages;	405	loff_t first_data_page, last_data_page, num_pages;
406	int shmem_page_offset;	406	int shmem_page_offset;
407	int data_page_index, data_page_offset;	407	int data_page_index, data_page_offset;
408	int page_length;	408	int page_length;
409	int ret;	409	int ret;
410	uint64_t data_ptr = args->data_ptr;	410	uint64_t data_ptr = args->data_ptr;
411	int do_bit17_swizzling;	411	int do_bit17_swizzling;
412		412
413	remain = args->size;	413	remain = args->size;
414		414
415	/* Pin the user pages containing the data. We can't fault while	415	/* Pin the user pages containing the data. We can't fault while
416	* holding the struct mutex, yet we want to hold it while	416	* holding the struct mutex, yet we want to hold it while
417	* dereferencing the user data.	417	* dereferencing the user data.
418	*/	418	*/
419	first_data_page = data_ptr / PAGE_SIZE;	419	first_data_page = data_ptr / PAGE_SIZE;
420	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;	420	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
421	num_pages = last_data_page - first_data_page + 1;	421	num_pages = last_data_page - first_data_page + 1;
422		422
423	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));	423	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
424	if (user_pages == NULL)	424	if (user_pages == NULL)
425	return -ENOMEM;	425	return -ENOMEM;
426		426
427	mutex_unlock(&dev->struct_mutex);	427	mutex_unlock(&dev->struct_mutex);
428	down_read(&mm->mmap_sem);	428	down_read(&mm->mmap_sem);
429	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,	429	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
430	num_pages, 1, 0, user_pages, NULL);	430	num_pages, 1, 0, user_pages, NULL);
431	up_read(&mm->mmap_sem);	431	up_read(&mm->mmap_sem);
432	mutex_lock(&dev->struct_mutex);	432	mutex_lock(&dev->struct_mutex);
433	if (pinned_pages < num_pages) {	433	if (pinned_pages < num_pages) {
434	ret = -EFAULT;	434	ret = -EFAULT;
435	goto out;	435	goto out;
436	}	436	}
437		437
438	ret = i915_gem_object_set_cpu_read_domain_range(obj,	438	ret = i915_gem_object_set_cpu_read_domain_range(obj,
439	args->offset,	439	args->offset,
440	args->size);	440	args->size);
441	if (ret)	441	if (ret)
442	goto out;	442	goto out;
443		443
444	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);	444	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
445		445
446	offset = args->offset;	446	offset = args->offset;
447		447
448	while (remain > 0) {	448	while (remain > 0) {
449	struct page *page;	449	struct page *page;
450		450
451	/* Operation in this page	451	/* Operation in this page
452	*	452	*
453	* shmem_page_offset = offset within page in shmem file	453	* shmem_page_offset = offset within page in shmem file
454	* data_page_index = page number in get_user_pages return	454	* data_page_index = page number in get_user_pages return
455	* data_page_offset = offset with data_page_index page.	455	* data_page_offset = offset with data_page_index page.
456	* page_length = bytes to copy for this page	456	* page_length = bytes to copy for this page
457	*/	457	*/
458	shmem_page_offset = offset_in_page(offset);	458	shmem_page_offset = offset_in_page(offset);
459	data_page_index = data_ptr / PAGE_SIZE - first_data_page;	459	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
460	data_page_offset = offset_in_page(data_ptr);	460	data_page_offset = offset_in_page(data_ptr);
461		461
462	page_length = remain;	462	page_length = remain;
463	if ((shmem_page_offset + page_length) > PAGE_SIZE)	463	if ((shmem_page_offset + page_length) > PAGE_SIZE)
464	page_length = PAGE_SIZE - shmem_page_offset;	464	page_length = PAGE_SIZE - shmem_page_offset;
465	if ((data_page_offset + page_length) > PAGE_SIZE)	465	if ((data_page_offset + page_length) > PAGE_SIZE)
466	page_length = PAGE_SIZE - data_page_offset;	466	page_length = PAGE_SIZE - data_page_offset;
467		467
468	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);	468	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
469	if (IS_ERR(page)) {	469	if (IS_ERR(page)) {
470	ret = PTR_ERR(page);	470	ret = PTR_ERR(page);
471	goto out;	471	goto out;
472	}	472	}
473		473
474	if (do_bit17_swizzling) {	474	if (do_bit17_swizzling) {
475	slow_shmem_bit17_copy(page,	475	slow_shmem_bit17_copy(page,
476	shmem_page_offset,	476	shmem_page_offset,
477	user_pages[data_page_index],	477	user_pages[data_page_index],
478	data_page_offset,	478	data_page_offset,
479	page_length,	479	page_length,
480	1);	480	1);
481	} else {	481	} else {
482	slow_shmem_copy(user_pages[data_page_index],	482	slow_shmem_copy(user_pages[data_page_index],
483	data_page_offset,	483	data_page_offset,
484	page,	484	page,
485	shmem_page_offset,	485	shmem_page_offset,
486	page_length);	486	page_length);
487	}	487	}
488		488
489	mark_page_accessed(page);	489	mark_page_accessed(page);
490	page_cache_release(page);	490	page_cache_release(page);
491		491
492	remain -= page_length;	492	remain -= page_length;
493	data_ptr += page_length;	493	data_ptr += page_length;
494	offset += page_length;	494	offset += page_length;
495	}	495	}
496		496
497	out:	497	out:
498	for (i = 0; i < pinned_pages; i++) {	498	for (i = 0; i < pinned_pages; i++) {
499	SetPageDirty(user_pages[i]);	499	SetPageDirty(user_pages[i]);
500	mark_page_accessed(user_pages[i]);	500	mark_page_accessed(user_pages[i]);
501	page_cache_release(user_pages[i]);	501	page_cache_release(user_pages[i]);
502	}	502	}
503	drm_free_large(user_pages);	503	drm_free_large(user_pages);
504		504
505	return ret;	505	return ret;
506	}	506	}
507		507
508	/**	508	/**
509	* Reads data from the object referenced by handle.	509	* Reads data from the object referenced by handle.
510	*	510	*
511	* On error, the contents of *data are undefined.	511	* On error, the contents of *data are undefined.
512	*/	512	*/
513	int	513	int
514	i915_gem_pread_ioctl(struct drm_device dev, void data,	514	i915_gem_pread_ioctl(struct drm_device dev, void data,
515	struct drm_file *file)	515	struct drm_file *file)
516	{	516	{
517	struct drm_i915_gem_pread *args = data;	517	struct drm_i915_gem_pread *args = data;
518	struct drm_i915_gem_object *obj;	518	struct drm_i915_gem_object *obj;
519	int ret = 0;	519	int ret = 0;
520		520
521	if (args->size == 0)	521	if (args->size == 0)
522	return 0;	522	return 0;
523		523
524	if (!access_ok(VERIFY_WRITE,	524	if (!access_ok(VERIFY_WRITE,
525	(char __user *)(uintptr_t)args->data_ptr,	525	(char __user *)(uintptr_t)args->data_ptr,
526	args->size))	526	args->size))
527	return -EFAULT;	527	return -EFAULT;
528		528
529	ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,	529	ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
530	args->size);	530	args->size);
531	if (ret)	531	if (ret)
532	return -EFAULT;	532	return -EFAULT;
533		533
534	ret = i915_mutex_lock_interruptible(dev);	534	ret = i915_mutex_lock_interruptible(dev);
535	if (ret)	535	if (ret)
536	return ret;	536	return ret;
537		537
538	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	538	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
539	if (&obj->base == NULL) {	539	if (&obj->base == NULL) {
540	ret = -ENOENT;	540	ret = -ENOENT;
541	goto unlock;	541	goto unlock;
542	}	542	}
543		543
544	/* Bounds check source. */	544	/* Bounds check source. */
545	if (args->offset > obj->base.size \|\|	545	if (args->offset > obj->base.size \|\|
546	args->size > obj->base.size - args->offset) {	546	args->size > obj->base.size - args->offset) {
547	ret = -EINVAL;	547	ret = -EINVAL;
548	goto out;	548	goto out;
549	}	549	}
550		550
551	trace_i915_gem_object_pread(obj, args->offset, args->size);	551	trace_i915_gem_object_pread(obj, args->offset, args->size);
552		552
553	ret = i915_gem_object_set_cpu_read_domain_range(obj,	553	ret = i915_gem_object_set_cpu_read_domain_range(obj,
554	args->offset,	554	args->offset,
555	args->size);	555	args->size);
556	if (ret)	556	if (ret)
557	goto out;	557	goto out;
558		558
559	ret = -EFAULT;	559	ret = -EFAULT;
560	if (!i915_gem_object_needs_bit17_swizzle(obj))	560	if (!i915_gem_object_needs_bit17_swizzle(obj))
561	ret = i915_gem_shmem_pread_fast(dev, obj, args, file);	561	ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
562	if (ret == -EFAULT)	562	if (ret == -EFAULT)
563	ret = i915_gem_shmem_pread_slow(dev, obj, args, file);	563	ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
564		564
565	out:	565	out:
566	drm_gem_object_unreference(&obj->base);	566	drm_gem_object_unreference(&obj->base);
567	unlock:	567	unlock:
568	mutex_unlock(&dev->struct_mutex);	568	mutex_unlock(&dev->struct_mutex);
569	return ret;	569	return ret;
570	}	570	}
571		571
572	/* This is the fast write path which cannot handle	572	/* This is the fast write path which cannot handle
573	* page faults in the source data	573	* page faults in the source data
574	*/	574	*/
575		575
576	static inline int	576	static inline int
577	fast_user_write(struct io_mapping *mapping,	577	fast_user_write(struct io_mapping *mapping,
578	loff_t page_base, int page_offset,	578	loff_t page_base, int page_offset,
579	char __user *user_data,	579	char __user *user_data,
580	int length)	580	int length)
581	{	581	{
582	char *vaddr_atomic;	582	char *vaddr_atomic;
583	unsigned long unwritten;	583	unsigned long unwritten;
584		584
585	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);	585	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
586	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,	586	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
587	user_data, length);	587	user_data, length);
588	io_mapping_unmap_atomic(vaddr_atomic);	588	io_mapping_unmap_atomic(vaddr_atomic);
589	return unwritten;	589	return unwritten;
590	}	590	}
591		591
592	/* Here's the write path which can sleep for	592	/* Here's the write path which can sleep for
593	* page faults	593	* page faults
594	*/	594	*/
595		595
596	static inline void	596	static inline void
597	slow_kernel_write(struct io_mapping *mapping,	597	slow_kernel_write(struct io_mapping *mapping,
598	loff_t gtt_base, int gtt_offset,	598	loff_t gtt_base, int gtt_offset,
599	struct page *user_page, int user_offset,	599	struct page *user_page, int user_offset,
600	int length)	600	int length)
601	{	601	{
602	char __iomem *dst_vaddr;	602	char __iomem *dst_vaddr;
603	char *src_vaddr;	603	char *src_vaddr;
604		604
605	dst_vaddr = io_mapping_map_wc(mapping, gtt_base);	605	dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
606	src_vaddr = kmap(user_page);	606	src_vaddr = kmap(user_page);
607		607
608	memcpy_toio(dst_vaddr + gtt_offset,	608	memcpy_toio(dst_vaddr + gtt_offset,
609	src_vaddr + user_offset,	609	src_vaddr + user_offset,
610	length);	610	length);
611		611
612	kunmap(user_page);	612	kunmap(user_page);
613	io_mapping_unmap(dst_vaddr);	613	io_mapping_unmap(dst_vaddr);
614	}	614	}
615		615
616	/**	616	/**
617	* This is the fast pwrite path, where we copy the data directly from the	617	* This is the fast pwrite path, where we copy the data directly from the
618	* user into the GTT, uncached.	618	* user into the GTT, uncached.
619	*/	619	*/
620	static int	620	static int
621	i915_gem_gtt_pwrite_fast(struct drm_device *dev,	621	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
622	struct drm_i915_gem_object *obj,	622	struct drm_i915_gem_object *obj,
623	struct drm_i915_gem_pwrite *args,	623	struct drm_i915_gem_pwrite *args,
624	struct drm_file *file)	624	struct drm_file *file)
625	{	625	{
626	drm_i915_private_t *dev_priv = dev->dev_private;	626	drm_i915_private_t *dev_priv = dev->dev_private;
627	ssize_t remain;	627	ssize_t remain;
628	loff_t offset, page_base;	628	loff_t offset, page_base;
629	char __user *user_data;	629	char __user *user_data;
630	int page_offset, page_length;	630	int page_offset, page_length;
631		631
632	user_data = (char __user *) (uintptr_t) args->data_ptr;	632	user_data = (char __user *) (uintptr_t) args->data_ptr;
633	remain = args->size;	633	remain = args->size;
634		634
635	offset = obj->gtt_offset + args->offset;	635	offset = obj->gtt_offset + args->offset;
636		636
637	while (remain > 0) {	637	while (remain > 0) {
638	/* Operation in this page	638	/* Operation in this page
639	*	639	*
640	* page_base = page offset within aperture	640	* page_base = page offset within aperture
641	* page_offset = offset within page	641	* page_offset = offset within page
642	* page_length = bytes to copy for this page	642	* page_length = bytes to copy for this page
643	*/	643	*/
644	page_base = offset & PAGE_MASK;	644	page_base = offset & PAGE_MASK;
645	page_offset = offset_in_page(offset);	645	page_offset = offset_in_page(offset);
646	page_length = remain;	646	page_length = remain;
647	if ((page_offset + remain) > PAGE_SIZE)	647	if ((page_offset + remain) > PAGE_SIZE)
648	page_length = PAGE_SIZE - page_offset;	648	page_length = PAGE_SIZE - page_offset;
649		649
650	/* If we get a fault while copying data, then (presumably) our	650	/* If we get a fault while copying data, then (presumably) our
651	* source page isn't available. Return the error and we'll	651	* source page isn't available. Return the error and we'll
652	* retry in the slow path.	652	* retry in the slow path.
653	*/	653	*/
654	if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,	654	if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
655	page_offset, user_data, page_length))	655	page_offset, user_data, page_length))
656	return -EFAULT;	656	return -EFAULT;
657		657
658	remain -= page_length;	658	remain -= page_length;
659	user_data += page_length;	659	user_data += page_length;
660	offset += page_length;	660	offset += page_length;
661	}	661	}
662		662
663	return 0;	663	return 0;
664	}	664	}
665		665
666	/**	666	/**
667	* This is the fallback GTT pwrite path, which uses get_user_pages to pin	667	* This is the fallback GTT pwrite path, which uses get_user_pages to pin
668	* the memory and maps it using kmap_atomic for copying.	668	* the memory and maps it using kmap_atomic for copying.
669	*	669	*
670	* This code resulted in x11perf -rgb10text consuming about 10% more CPU	670	* This code resulted in x11perf -rgb10text consuming about 10% more CPU
671	* than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).	671	* than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
672	*/	672	*/
673	static int	673	static int
674	i915_gem_gtt_pwrite_slow(struct drm_device *dev,	674	i915_gem_gtt_pwrite_slow(struct drm_device *dev,
675	struct drm_i915_gem_object *obj,	675	struct drm_i915_gem_object *obj,
676	struct drm_i915_gem_pwrite *args,	676	struct drm_i915_gem_pwrite *args,
677	struct drm_file *file)	677	struct drm_file *file)
678	{	678	{
679	drm_i915_private_t *dev_priv = dev->dev_private;	679	drm_i915_private_t *dev_priv = dev->dev_private;
680	ssize_t remain;	680	ssize_t remain;
681	loff_t gtt_page_base, offset;	681	loff_t gtt_page_base, offset;
682	loff_t first_data_page, last_data_page, num_pages;	682	loff_t first_data_page, last_data_page, num_pages;
683	loff_t pinned_pages, i;	683	loff_t pinned_pages, i;
684	struct page **user_pages;	684	struct page **user_pages;
685	struct mm_struct *mm = current->mm;	685	struct mm_struct *mm = current->mm;
686	int gtt_page_offset, data_page_offset, data_page_index, page_length;	686	int gtt_page_offset, data_page_offset, data_page_index, page_length;
687	int ret;	687	int ret;
688	uint64_t data_ptr = args->data_ptr;	688	uint64_t data_ptr = args->data_ptr;
689		689
690	remain = args->size;	690	remain = args->size;
691		691
692	/* Pin the user pages containing the data. We can't fault while	692	/* Pin the user pages containing the data. We can't fault while
693	* holding the struct mutex, and all of the pwrite implementations	693	* holding the struct mutex, and all of the pwrite implementations
694	* want to hold it while dereferencing the user data.	694	* want to hold it while dereferencing the user data.
695	*/	695	*/
696	first_data_page = data_ptr / PAGE_SIZE;	696	first_data_page = data_ptr / PAGE_SIZE;
697	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;	697	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
698	num_pages = last_data_page - first_data_page + 1;	698	num_pages = last_data_page - first_data_page + 1;
699		699
700	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));	700	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
701	if (user_pages == NULL)	701	if (user_pages == NULL)
702	return -ENOMEM;	702	return -ENOMEM;
703		703
704	mutex_unlock(&dev->struct_mutex);	704	mutex_unlock(&dev->struct_mutex);
705	down_read(&mm->mmap_sem);	705	down_read(&mm->mmap_sem);
706	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,	706	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
707	num_pages, 0, 0, user_pages, NULL);	707	num_pages, 0, 0, user_pages, NULL);
708	up_read(&mm->mmap_sem);	708	up_read(&mm->mmap_sem);
709	mutex_lock(&dev->struct_mutex);	709	mutex_lock(&dev->struct_mutex);
710	if (pinned_pages < num_pages) {	710	if (pinned_pages < num_pages) {
711	ret = -EFAULT;	711	ret = -EFAULT;
712	goto out_unpin_pages;	712	goto out_unpin_pages;
713	}	713	}
714		714
715	ret = i915_gem_object_set_to_gtt_domain(obj, true);	715	ret = i915_gem_object_set_to_gtt_domain(obj, true);
716	if (ret)	716	if (ret)
717	goto out_unpin_pages;	717	goto out_unpin_pages;
718		718
719	ret = i915_gem_object_put_fence(obj);	719	ret = i915_gem_object_put_fence(obj);
720	if (ret)	720	if (ret)
721	goto out_unpin_pages;	721	goto out_unpin_pages;
722		722
723	offset = obj->gtt_offset + args->offset;	723	offset = obj->gtt_offset + args->offset;
724		724
725	while (remain > 0) {	725	while (remain > 0) {
726	/* Operation in this page	726	/* Operation in this page
727	*	727	*
728	* gtt_page_base = page offset within aperture	728	* gtt_page_base = page offset within aperture
729	* gtt_page_offset = offset within page in aperture	729	* gtt_page_offset = offset within page in aperture
730	* data_page_index = page number in get_user_pages return	730	* data_page_index = page number in get_user_pages return
731	* data_page_offset = offset with data_page_index page.	731	* data_page_offset = offset with data_page_index page.
732	* page_length = bytes to copy for this page	732	* page_length = bytes to copy for this page
733	*/	733	*/
734	gtt_page_base = offset & PAGE_MASK;	734	gtt_page_base = offset & PAGE_MASK;
735	gtt_page_offset = offset_in_page(offset);	735	gtt_page_offset = offset_in_page(offset);
736	data_page_index = data_ptr / PAGE_SIZE - first_data_page;	736	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
737	data_page_offset = offset_in_page(data_ptr);	737	data_page_offset = offset_in_page(data_ptr);
738		738
739	page_length = remain;	739	page_length = remain;
740	if ((gtt_page_offset + page_length) > PAGE_SIZE)	740	if ((gtt_page_offset + page_length) > PAGE_SIZE)
741	page_length = PAGE_SIZE - gtt_page_offset;	741	page_length = PAGE_SIZE - gtt_page_offset;
742	if ((data_page_offset + page_length) > PAGE_SIZE)	742	if ((data_page_offset + page_length) > PAGE_SIZE)
743	page_length = PAGE_SIZE - data_page_offset;	743	page_length = PAGE_SIZE - data_page_offset;
744		744
745	slow_kernel_write(dev_priv->mm.gtt_mapping,	745	slow_kernel_write(dev_priv->mm.gtt_mapping,
746	gtt_page_base, gtt_page_offset,	746	gtt_page_base, gtt_page_offset,
747	user_pages[data_page_index],	747	user_pages[data_page_index],
748	data_page_offset,	748	data_page_offset,
749	page_length);	749	page_length);
750		750
751	remain -= page_length;	751	remain -= page_length;
752	offset += page_length;	752	offset += page_length;
753	data_ptr += page_length;	753	data_ptr += page_length;
754	}	754	}
755		755
756	out_unpin_pages:	756	out_unpin_pages:
757	for (i = 0; i < pinned_pages; i++)	757	for (i = 0; i < pinned_pages; i++)
758	page_cache_release(user_pages[i]);	758	page_cache_release(user_pages[i]);
759	drm_free_large(user_pages);	759	drm_free_large(user_pages);
760		760
761	return ret;	761	return ret;
762	}	762	}
763		763
764	/**	764	/**
765	* This is the fast shmem pwrite path, which attempts to directly	765	* This is the fast shmem pwrite path, which attempts to directly
766	* copy_from_user into the kmapped pages backing the object.	766	* copy_from_user into the kmapped pages backing the object.
767	*/	767	*/
768	static int	768	static int
769	i915_gem_shmem_pwrite_fast(struct drm_device *dev,	769	i915_gem_shmem_pwrite_fast(struct drm_device *dev,
770	struct drm_i915_gem_object *obj,	770	struct drm_i915_gem_object *obj,
771	struct drm_i915_gem_pwrite *args,	771	struct drm_i915_gem_pwrite *args,
772	struct drm_file *file)	772	struct drm_file *file)
773	{	773	{
774	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	774	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
775	ssize_t remain;	775	ssize_t remain;
776	loff_t offset;	776	loff_t offset;
777	char __user *user_data;	777	char __user *user_data;
778	int page_offset, page_length;	778	int page_offset, page_length;
779		779
780	user_data = (char __user *) (uintptr_t) args->data_ptr;	780	user_data = (char __user *) (uintptr_t) args->data_ptr;
781	remain = args->size;	781	remain = args->size;
782		782
783	offset = args->offset;	783	offset = args->offset;
784	obj->dirty = 1;	784	obj->dirty = 1;
785		785
786	while (remain > 0) {	786	while (remain > 0) {
787	struct page *page;	787	struct page *page;
788	char *vaddr;	788	char *vaddr;
789	int ret;	789	int ret;
790		790
791	/* Operation in this page	791	/* Operation in this page
792	*	792	*
793	* page_offset = offset within page	793	* page_offset = offset within page
794	* page_length = bytes to copy for this page	794	* page_length = bytes to copy for this page
795	*/	795	*/
796	page_offset = offset_in_page(offset);	796	page_offset = offset_in_page(offset);
797	page_length = remain;	797	page_length = remain;
798	if ((page_offset + remain) > PAGE_SIZE)	798	if ((page_offset + remain) > PAGE_SIZE)
799	page_length = PAGE_SIZE - page_offset;	799	page_length = PAGE_SIZE - page_offset;
800		800
801	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);	801	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
802	if (IS_ERR(page))	802	if (IS_ERR(page))
803	return PTR_ERR(page);	803	return PTR_ERR(page);
804		804
805	vaddr = kmap_atomic(page);	805	vaddr = kmap_atomic(page);
806	ret = __copy_from_user_inatomic(vaddr + page_offset,	806	ret = __copy_from_user_inatomic(vaddr + page_offset,
807	user_data,	807	user_data,
808	page_length);	808	page_length);
809	kunmap_atomic(vaddr);	809	kunmap_atomic(vaddr);
810		810
811	set_page_dirty(page);	811	set_page_dirty(page);
812	mark_page_accessed(page);	812	mark_page_accessed(page);
813	page_cache_release(page);	813	page_cache_release(page);
814		814
815	/* If we get a fault while copying data, then (presumably) our	815	/* If we get a fault while copying data, then (presumably) our
816	* source page isn't available. Return the error and we'll	816	* source page isn't available. Return the error and we'll
817	* retry in the slow path.	817	* retry in the slow path.
818	*/	818	*/
819	if (ret)	819	if (ret)
820	return -EFAULT;	820	return -EFAULT;
821		821
822	remain -= page_length;	822	remain -= page_length;
823	user_data += page_length;	823	user_data += page_length;
824	offset += page_length;	824	offset += page_length;
825	}	825	}
826		826
827	return 0;	827	return 0;
828	}	828	}
829		829
830	/**	830	/**
831	* This is the fallback shmem pwrite path, which uses get_user_pages to pin	831	* This is the fallback shmem pwrite path, which uses get_user_pages to pin
832	* the memory and maps it using kmap_atomic for copying.	832	* the memory and maps it using kmap_atomic for copying.
833	*	833	*
834	* This avoids taking mmap_sem for faulting on the user's address while the	834	* This avoids taking mmap_sem for faulting on the user's address while the
835	* struct_mutex is held.	835	* struct_mutex is held.
836	*/	836	*/
837	static int	837	static int
838	i915_gem_shmem_pwrite_slow(struct drm_device *dev,	838	i915_gem_shmem_pwrite_slow(struct drm_device *dev,
839	struct drm_i915_gem_object *obj,	839	struct drm_i915_gem_object *obj,
840	struct drm_i915_gem_pwrite *args,	840	struct drm_i915_gem_pwrite *args,
841	struct drm_file *file)	841	struct drm_file *file)
842	{	842	{
843	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	843	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
844	struct mm_struct *mm = current->mm;	844	struct mm_struct *mm = current->mm;
845	struct page **user_pages;	845	struct page **user_pages;
846	ssize_t remain;	846	ssize_t remain;
847	loff_t offset, pinned_pages, i;	847	loff_t offset, pinned_pages, i;
848	loff_t first_data_page, last_data_page, num_pages;	848	loff_t first_data_page, last_data_page, num_pages;
849	int shmem_page_offset;	849	int shmem_page_offset;
850	int data_page_index, data_page_offset;	850	int data_page_index, data_page_offset;
851	int page_length;	851	int page_length;
852	int ret;	852	int ret;
853	uint64_t data_ptr = args->data_ptr;	853	uint64_t data_ptr = args->data_ptr;
854	int do_bit17_swizzling;	854	int do_bit17_swizzling;
855		855
856	remain = args->size;	856	remain = args->size;
857		857
858	/* Pin the user pages containing the data. We can't fault while	858	/* Pin the user pages containing the data. We can't fault while
859	* holding the struct mutex, and all of the pwrite implementations	859	* holding the struct mutex, and all of the pwrite implementations
860	* want to hold it while dereferencing the user data.	860	* want to hold it while dereferencing the user data.
861	*/	861	*/
862	first_data_page = data_ptr / PAGE_SIZE;	862	first_data_page = data_ptr / PAGE_SIZE;
863	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;	863	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
864	num_pages = last_data_page - first_data_page + 1;	864	num_pages = last_data_page - first_data_page + 1;
865		865
866	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));	866	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
867	if (user_pages == NULL)	867	if (user_pages == NULL)
868	return -ENOMEM;	868	return -ENOMEM;
869		869
870	mutex_unlock(&dev->struct_mutex);	870	mutex_unlock(&dev->struct_mutex);
871	down_read(&mm->mmap_sem);	871	down_read(&mm->mmap_sem);
872	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,	872	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
873	num_pages, 0, 0, user_pages, NULL);	873	num_pages, 0, 0, user_pages, NULL);
874	up_read(&mm->mmap_sem);	874	up_read(&mm->mmap_sem);
875	mutex_lock(&dev->struct_mutex);	875	mutex_lock(&dev->struct_mutex);
876	if (pinned_pages < num_pages) {	876	if (pinned_pages < num_pages) {
877	ret = -EFAULT;	877	ret = -EFAULT;
878	goto out;	878	goto out;
879	}	879	}
880		880
881	ret = i915_gem_object_set_to_cpu_domain(obj, 1);	881	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
882	if (ret)	882	if (ret)
883	goto out;	883	goto out;
884		884
885	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);	885	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
886		886
887	offset = args->offset;	887	offset = args->offset;
888	obj->dirty = 1;	888	obj->dirty = 1;
889		889
890	while (remain > 0) {	890	while (remain > 0) {
891	struct page *page;	891	struct page *page;
892		892
893	/* Operation in this page	893	/* Operation in this page
894	*	894	*
895	* shmem_page_offset = offset within page in shmem file	895	* shmem_page_offset = offset within page in shmem file
896	* data_page_index = page number in get_user_pages return	896	* data_page_index = page number in get_user_pages return
897	* data_page_offset = offset with data_page_index page.	897	* data_page_offset = offset with data_page_index page.
898	* page_length = bytes to copy for this page	898	* page_length = bytes to copy for this page
899	*/	899	*/
900	shmem_page_offset = offset_in_page(offset);	900	shmem_page_offset = offset_in_page(offset);
901	data_page_index = data_ptr / PAGE_SIZE - first_data_page;	901	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
902	data_page_offset = offset_in_page(data_ptr);	902	data_page_offset = offset_in_page(data_ptr);
903		903
904	page_length = remain;	904	page_length = remain;
905	if ((shmem_page_offset + page_length) > PAGE_SIZE)	905	if ((shmem_page_offset + page_length) > PAGE_SIZE)
906	page_length = PAGE_SIZE - shmem_page_offset;	906	page_length = PAGE_SIZE - shmem_page_offset;
907	if ((data_page_offset + page_length) > PAGE_SIZE)	907	if ((data_page_offset + page_length) > PAGE_SIZE)
908	page_length = PAGE_SIZE - data_page_offset;	908	page_length = PAGE_SIZE - data_page_offset;
909		909
910	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);	910	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
911	if (IS_ERR(page)) {	911	if (IS_ERR(page)) {
912	ret = PTR_ERR(page);	912	ret = PTR_ERR(page);
913	goto out;	913	goto out;
914	}	914	}
915		915
916	if (do_bit17_swizzling) {	916	if (do_bit17_swizzling) {
917	slow_shmem_bit17_copy(page,	917	slow_shmem_bit17_copy(page,
918	shmem_page_offset,	918	shmem_page_offset,
919	user_pages[data_page_index],	919	user_pages[data_page_index],
920	data_page_offset,	920	data_page_offset,
921	page_length,	921	page_length,
922	0);	922	0);
923	} else {	923	} else {
924	slow_shmem_copy(page,	924	slow_shmem_copy(page,
925	shmem_page_offset,	925	shmem_page_offset,
926	user_pages[data_page_index],	926	user_pages[data_page_index],
927	data_page_offset,	927	data_page_offset,
928	page_length);	928	page_length);
929	}	929	}
930		930
931	set_page_dirty(page);	931	set_page_dirty(page);
932	mark_page_accessed(page);	932	mark_page_accessed(page);
933	page_cache_release(page);	933	page_cache_release(page);
934		934
935	remain -= page_length;	935	remain -= page_length;
936	data_ptr += page_length;	936	data_ptr += page_length;
937	offset += page_length;	937	offset += page_length;
938	}	938	}
939		939
940	out:	940	out:
941	for (i = 0; i < pinned_pages; i++)	941	for (i = 0; i < pinned_pages; i++)
942	page_cache_release(user_pages[i]);	942	page_cache_release(user_pages[i]);
943	drm_free_large(user_pages);	943	drm_free_large(user_pages);
944		944
945	return ret;	945	return ret;
946	}	946	}
947		947
948	/**	948	/**
949	* Writes data to the object referenced by handle.	949	* Writes data to the object referenced by handle.
950	*	950	*
951	* On error, the contents of the buffer that were to be modified are undefined.	951	* On error, the contents of the buffer that were to be modified are undefined.
952	*/	952	*/
953	int	953	int
954	i915_gem_pwrite_ioctl(struct drm_device dev, void data,	954	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
955	struct drm_file *file)	955	struct drm_file *file)
956	{	956	{
957	struct drm_i915_gem_pwrite *args = data;	957	struct drm_i915_gem_pwrite *args = data;
958	struct drm_i915_gem_object *obj;	958	struct drm_i915_gem_object *obj;
959	int ret;	959	int ret;
960		960
961	if (args->size == 0)	961	if (args->size == 0)
962	return 0;	962	return 0;
963		963
964	if (!access_ok(VERIFY_READ,	964	if (!access_ok(VERIFY_READ,
965	(char __user *)(uintptr_t)args->data_ptr,	965	(char __user *)(uintptr_t)args->data_ptr,
966	args->size))	966	args->size))
967	return -EFAULT;	967	return -EFAULT;
968		968
969	ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,	969	ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
970	args->size);	970	args->size);
971	if (ret)	971	if (ret)
972	return -EFAULT;	972	return -EFAULT;
973		973
974	ret = i915_mutex_lock_interruptible(dev);	974	ret = i915_mutex_lock_interruptible(dev);
975	if (ret)	975	if (ret)
976	return ret;	976	return ret;
977		977
978	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	978	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
979	if (&obj->base == NULL) {	979	if (&obj->base == NULL) {
980	ret = -ENOENT;	980	ret = -ENOENT;
981	goto unlock;	981	goto unlock;
982	}	982	}
983		983
984	/* Bounds check destination. */	984	/* Bounds check destination. */
985	if (args->offset > obj->base.size \|\|	985	if (args->offset > obj->base.size \|\|
986	args->size > obj->base.size - args->offset) {	986	args->size > obj->base.size - args->offset) {
987	ret = -EINVAL;	987	ret = -EINVAL;
988	goto out;	988	goto out;
989	}	989	}
990		990
991	trace_i915_gem_object_pwrite(obj, args->offset, args->size);	991	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
992		992
993	/* We can only do the GTT pwrite on untiled buffers, as otherwise	993	/* We can only do the GTT pwrite on untiled buffers, as otherwise
994	* it would end up going through the fenced access, and we'll get	994	* it would end up going through the fenced access, and we'll get
995	* different detiling behavior between reading and writing.	995	* different detiling behavior between reading and writing.
996	* pread/pwrite currently are reading and writing from the CPU	996	* pread/pwrite currently are reading and writing from the CPU
997	* perspective, requiring manual detiling by the client.	997	* perspective, requiring manual detiling by the client.
998	*/	998	*/
999	if (obj->phys_obj)	999	if (obj->phys_obj)
1000	ret = i915_gem_phys_pwrite(dev, obj, args, file);	1000	ret = i915_gem_phys_pwrite(dev, obj, args, file);
1001	else if (obj->gtt_space &&	1001	else if (obj->gtt_space &&
1002	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {	1002	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1003	ret = i915_gem_object_pin(obj, 0, true);	1003	ret = i915_gem_object_pin(obj, 0, true);
1004	if (ret)	1004	if (ret)
1005	goto out;	1005	goto out;
1006		1006
1007	ret = i915_gem_object_set_to_gtt_domain(obj, true);	1007	ret = i915_gem_object_set_to_gtt_domain(obj, true);
1008	if (ret)	1008	if (ret)
1009	goto out_unpin;	1009	goto out_unpin;
1010		1010
1011	ret = i915_gem_object_put_fence(obj);	1011	ret = i915_gem_object_put_fence(obj);
1012	if (ret)	1012	if (ret)
1013	goto out_unpin;	1013	goto out_unpin;
1014		1014
1015	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);	1015	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1016	if (ret == -EFAULT)	1016	if (ret == -EFAULT)
1017	ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);	1017	ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1018		1018
1019	out_unpin:	1019	out_unpin:
1020	i915_gem_object_unpin(obj);	1020	i915_gem_object_unpin(obj);
1021	} else {	1021	} else {
1022	ret = i915_gem_object_set_to_cpu_domain(obj, 1);	1022	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1023	if (ret)	1023	if (ret)
1024	goto out;	1024	goto out;
1025		1025
1026	ret = -EFAULT;	1026	ret = -EFAULT;
1027	if (!i915_gem_object_needs_bit17_swizzle(obj))	1027	if (!i915_gem_object_needs_bit17_swizzle(obj))
1028	ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);	1028	ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1029	if (ret == -EFAULT)	1029	if (ret == -EFAULT)
1030	ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);	1030	ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1031	}	1031	}
1032		1032
1033	out:	1033	out:
1034	drm_gem_object_unreference(&obj->base);	1034	drm_gem_object_unreference(&obj->base);
1035	unlock:	1035	unlock:
1036	mutex_unlock(&dev->struct_mutex);	1036	mutex_unlock(&dev->struct_mutex);
1037	return ret;	1037	return ret;
1038	}	1038	}
1039		1039
1040	/**	1040	/**
1041	* Called when user space prepares to use an object with the CPU, either	1041	* Called when user space prepares to use an object with the CPU, either
1042	* through the mmap ioctl's mapping or a GTT mapping.	1042	* through the mmap ioctl's mapping or a GTT mapping.
1043	*/	1043	*/
1044	int	1044	int
1045	i915_gem_set_domain_ioctl(struct drm_device dev, void data,	1045	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
1046	struct drm_file *file)	1046	struct drm_file *file)
1047	{	1047	{
1048	struct drm_i915_gem_set_domain *args = data;	1048	struct drm_i915_gem_set_domain *args = data;
1049	struct drm_i915_gem_object *obj;	1049	struct drm_i915_gem_object *obj;
1050	uint32_t read_domains = args->read_domains;	1050	uint32_t read_domains = args->read_domains;
1051	uint32_t write_domain = args->write_domain;	1051	uint32_t write_domain = args->write_domain;
1052	int ret;	1052	int ret;
1053		1053
1054	if (!(dev->driver->driver_features & DRIVER_GEM))	1054	if (!(dev->driver->driver_features & DRIVER_GEM))
1055	return -ENODEV;	1055	return -ENODEV;
1056		1056
1057	/* Only handle setting domains to types used by the CPU. */	1057	/* Only handle setting domains to types used by the CPU. */
1058	if (write_domain & I915_GEM_GPU_DOMAINS)	1058	if (write_domain & I915_GEM_GPU_DOMAINS)
1059	return -EINVAL;	1059	return -EINVAL;
1060		1060
1061	if (read_domains & I915_GEM_GPU_DOMAINS)	1061	if (read_domains & I915_GEM_GPU_DOMAINS)
1062	return -EINVAL;	1062	return -EINVAL;
1063		1063
1064	/* Having something in the write domain implies it's in the read	1064	/* Having something in the write domain implies it's in the read
1065	* domain, and only that read domain. Enforce that in the request.	1065	* domain, and only that read domain. Enforce that in the request.
1066	*/	1066	*/
1067	if (write_domain != 0 && read_domains != write_domain)	1067	if (write_domain != 0 && read_domains != write_domain)
1068	return -EINVAL;	1068	return -EINVAL;
1069		1069
1070	ret = i915_mutex_lock_interruptible(dev);	1070	ret = i915_mutex_lock_interruptible(dev);
1071	if (ret)	1071	if (ret)
1072	return ret;	1072	return ret;
1073		1073
1074	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	1074	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1075	if (&obj->base == NULL) {	1075	if (&obj->base == NULL) {
1076	ret = -ENOENT;	1076	ret = -ENOENT;
1077	goto unlock;	1077	goto unlock;
1078	}	1078	}
1079		1079
1080	if (read_domains & I915_GEM_DOMAIN_GTT) {	1080	if (read_domains & I915_GEM_DOMAIN_GTT) {
1081	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);	1081	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1082		1082
1083	/* Silently promote "you're not bound, there was nothing to do"	1083	/* Silently promote "you're not bound, there was nothing to do"
1084	* to success, since the client was just asking us to	1084	* to success, since the client was just asking us to
1085	* make sure everything was done.	1085	* make sure everything was done.
1086	*/	1086	*/
1087	if (ret == -EINVAL)	1087	if (ret == -EINVAL)
1088	ret = 0;	1088	ret = 0;
1089	} else {	1089	} else {
1090	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);	1090	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1091	}	1091	}
1092		1092
1093	drm_gem_object_unreference(&obj->base);	1093	drm_gem_object_unreference(&obj->base);
1094	unlock:	1094	unlock:
1095	mutex_unlock(&dev->struct_mutex);	1095	mutex_unlock(&dev->struct_mutex);
1096	return ret;	1096	return ret;
1097	}	1097	}
1098		1098
1099	/**	1099	/**
1100	* Called when user space has done writes to this buffer	1100	* Called when user space has done writes to this buffer
1101	*/	1101	*/
1102	int	1102	int
1103	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,	1103	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
1104	struct drm_file *file)	1104	struct drm_file *file)
1105	{	1105	{
1106	struct drm_i915_gem_sw_finish *args = data;	1106	struct drm_i915_gem_sw_finish *args = data;
1107	struct drm_i915_gem_object *obj;	1107	struct drm_i915_gem_object *obj;
1108	int ret = 0;	1108	int ret = 0;
1109		1109
1110	if (!(dev->driver->driver_features & DRIVER_GEM))	1110	if (!(dev->driver->driver_features & DRIVER_GEM))
1111	return -ENODEV;	1111	return -ENODEV;
1112		1112
1113	ret = i915_mutex_lock_interruptible(dev);	1113	ret = i915_mutex_lock_interruptible(dev);
1114	if (ret)	1114	if (ret)
1115	return ret;	1115	return ret;
1116		1116
1117	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	1117	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1118	if (&obj->base == NULL) {	1118	if (&obj->base == NULL) {
1119	ret = -ENOENT;	1119	ret = -ENOENT;
1120	goto unlock;	1120	goto unlock;
1121	}	1121	}
1122		1122
1123	/* Pinned buffers may be scanout, so flush the cache */	1123	/* Pinned buffers may be scanout, so flush the cache */
1124	if (obj->pin_count)	1124	if (obj->pin_count)
1125	i915_gem_object_flush_cpu_write_domain(obj);	1125	i915_gem_object_flush_cpu_write_domain(obj);
1126		1126
1127	drm_gem_object_unreference(&obj->base);	1127	drm_gem_object_unreference(&obj->base);
1128	unlock:	1128	unlock:
1129	mutex_unlock(&dev->struct_mutex);	1129	mutex_unlock(&dev->struct_mutex);
1130	return ret;	1130	return ret;
1131	}	1131	}
1132		1132
1133	/**	1133	/**
1134	* Maps the contents of an object, returning the address it is mapped	1134	* Maps the contents of an object, returning the address it is mapped
1135	* into.	1135	* into.
1136	*	1136	*
1137	* While the mapping holds a reference on the contents of the object, it doesn't	1137	* While the mapping holds a reference on the contents of the object, it doesn't
1138	* imply a ref on the object itself.	1138	* imply a ref on the object itself.
1139	*/	1139	*/
1140	int	1140	int
1141	i915_gem_mmap_ioctl(struct drm_device dev, void data,	1141	i915_gem_mmap_ioctl(struct drm_device dev, void data,
1142	struct drm_file *file)	1142	struct drm_file *file)
1143	{	1143	{
1144	struct drm_i915_private *dev_priv = dev->dev_private;	1144	struct drm_i915_private *dev_priv = dev->dev_private;
1145	struct drm_i915_gem_mmap *args = data;	1145	struct drm_i915_gem_mmap *args = data;
1146	struct drm_gem_object *obj;	1146	struct drm_gem_object *obj;
1147	unsigned long addr;	1147	unsigned long addr;
1148		1148
1149	if (!(dev->driver->driver_features & DRIVER_GEM))	1149	if (!(dev->driver->driver_features & DRIVER_GEM))
1150	return -ENODEV;	1150	return -ENODEV;
1151		1151
1152	obj = drm_gem_object_lookup(dev, file, args->handle);	1152	obj = drm_gem_object_lookup(dev, file, args->handle);
1153	if (obj == NULL)	1153	if (obj == NULL)
1154	return -ENOENT;	1154	return -ENOENT;
1155		1155
1156	if (obj->size > dev_priv->mm.gtt_mappable_end) {	1156	if (obj->size > dev_priv->mm.gtt_mappable_end) {
1157	drm_gem_object_unreference_unlocked(obj);	1157	drm_gem_object_unreference_unlocked(obj);
1158	return -E2BIG;	1158	return -E2BIG;
1159	}	1159	}
1160		1160
1161	down_write(&current->mm->mmap_sem);	1161	down_write(&current->mm->mmap_sem);
1162	addr = do_mmap(obj->filp, 0, args->size,	1162	addr = do_mmap(obj->filp, 0, args->size,
1163	PROT_READ \| PROT_WRITE, MAP_SHARED,	1163	PROT_READ \| PROT_WRITE, MAP_SHARED,
1164	args->offset);	1164	args->offset);
1165	up_write(&current->mm->mmap_sem);	1165	up_write(&current->mm->mmap_sem);
1166	drm_gem_object_unreference_unlocked(obj);	1166	drm_gem_object_unreference_unlocked(obj);
1167	if (IS_ERR((void *)addr))	1167	if (IS_ERR((void *)addr))
1168	return addr;	1168	return addr;
1169		1169
1170	args->addr_ptr = (uint64_t) addr;	1170	args->addr_ptr = (uint64_t) addr;
1171		1171
1172	return 0;	1172	return 0;
1173	}	1173	}
1174		1174
1175	/**	1175	/**
1176	* i915_gem_fault - fault a page into the GTT	1176	* i915_gem_fault - fault a page into the GTT
1177	* vma: VMA in question	1177	* vma: VMA in question
1178	* vmf: fault info	1178	* vmf: fault info
1179	*	1179	*
1180	* The fault handler is set up by drm_gem_mmap() when a object is GTT mapped	1180	* The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1181	* from userspace. The fault handler takes care of binding the object to	1181	* from userspace. The fault handler takes care of binding the object to
1182	* the GTT (if needed), allocating and programming a fence register (again,	1182	* the GTT (if needed), allocating and programming a fence register (again,
1183	* only if needed based on whether the old reg is still valid or the object	1183	* only if needed based on whether the old reg is still valid or the object
1184	* is tiled) and inserting a new PTE into the faulting process.	1184	* is tiled) and inserting a new PTE into the faulting process.
1185	*	1185	*
1186	* Note that the faulting process may involve evicting existing objects	1186	* Note that the faulting process may involve evicting existing objects
1187	* from the GTT and/or fence registers to make room. So performance may	1187	* from the GTT and/or fence registers to make room. So performance may
1188	* suffer if the GTT working set is large or there are few fence registers	1188	* suffer if the GTT working set is large or there are few fence registers
1189	* left.	1189	* left.
1190	*/	1190	*/
1191	int i915_gem_fault(struct vm_area_struct vma, struct vm_fault vmf)	1191	int i915_gem_fault(struct vm_area_struct vma, struct vm_fault vmf)
1192	{	1192	{
1193	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);	1193	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1194	struct drm_device *dev = obj->base.dev;	1194	struct drm_device *dev = obj->base.dev;
1195	drm_i915_private_t *dev_priv = dev->dev_private;	1195	drm_i915_private_t *dev_priv = dev->dev_private;
1196	pgoff_t page_offset;	1196	pgoff_t page_offset;
1197	unsigned long pfn;	1197	unsigned long pfn;
1198	int ret = 0;	1198	int ret = 0;
1199	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);	1199	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1200		1200
1201	/* We don't use vmf->pgoff since that has the fake offset */	1201	/* We don't use vmf->pgoff since that has the fake offset */
1202	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>	1202	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1203	PAGE_SHIFT;	1203	PAGE_SHIFT;
1204		1204
1205	ret = i915_mutex_lock_interruptible(dev);	1205	ret = i915_mutex_lock_interruptible(dev);
1206	if (ret)	1206	if (ret)
1207	goto out;	1207	goto out;
1208		1208
1209	trace_i915_gem_object_fault(obj, page_offset, true, write);	1209	trace_i915_gem_object_fault(obj, page_offset, true, write);
1210		1210
1211	/* Now bind it into the GTT if needed */	1211	/* Now bind it into the GTT if needed */
1212	if (!obj->map_and_fenceable) {	1212	if (!obj->map_and_fenceable) {
1213	ret = i915_gem_object_unbind(obj);	1213	ret = i915_gem_object_unbind(obj);
1214	if (ret)	1214	if (ret)
1215	goto unlock;	1215	goto unlock;
1216	}	1216	}
1217	if (!obj->gtt_space) {	1217	if (!obj->gtt_space) {
1218	ret = i915_gem_object_bind_to_gtt(obj, 0, true);	1218	ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1219	if (ret)	1219	if (ret)
1220	goto unlock;	1220	goto unlock;
1221		1221
1222	ret = i915_gem_object_set_to_gtt_domain(obj, write);	1222	ret = i915_gem_object_set_to_gtt_domain(obj, write);
1223	if (ret)	1223	if (ret)
1224	goto unlock;	1224	goto unlock;
1225	}	1225	}
1226		1226
1227	if (obj->tiling_mode == I915_TILING_NONE)	1227	if (obj->tiling_mode == I915_TILING_NONE)
1228	ret = i915_gem_object_put_fence(obj);	1228	ret = i915_gem_object_put_fence(obj);
1229	else	1229	else
1230	ret = i915_gem_object_get_fence(obj, NULL);	1230	ret = i915_gem_object_get_fence(obj, NULL);
1231	if (ret)	1231	if (ret)
1232	goto unlock;	1232	goto unlock;
1233		1233
1234	if (i915_gem_object_is_inactive(obj))	1234	if (i915_gem_object_is_inactive(obj))
1235	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);	1235	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1236		1236
1237	obj->fault_mappable = true;	1237	obj->fault_mappable = true;
1238		1238
1239	pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +	1239	pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1240	page_offset;	1240	page_offset;
1241		1241
1242	/* Finally, remap it using the new GTT offset */	1242	/* Finally, remap it using the new GTT offset */
1243	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);	1243	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1244	unlock:	1244	unlock:
1245	mutex_unlock(&dev->struct_mutex);	1245	mutex_unlock(&dev->struct_mutex);
1246	out:	1246	out:
1247	switch (ret) {	1247	switch (ret) {
1248	case -EIO:	1248	case -EIO:
1249	case -EAGAIN:	1249	case -EAGAIN:
1250	/* Give the error handler a chance to run and move the	1250	/* Give the error handler a chance to run and move the
1251	* objects off the GPU active list. Next time we service the	1251	* objects off the GPU active list. Next time we service the
1252	* fault, we should be able to transition the page into the	1252	* fault, we should be able to transition the page into the
1253	* GTT without touching the GPU (and so avoid further	1253	* GTT without touching the GPU (and so avoid further
1254	* EIO/EGAIN). If the GPU is wedged, then there is no issue	1254	* EIO/EGAIN). If the GPU is wedged, then there is no issue
1255	* with coherency, just lost writes.	1255	* with coherency, just lost writes.
1256	*/	1256	*/
1257	set_need_resched();	1257	set_need_resched();
1258	case 0:	1258	case 0:
1259	case -ERESTARTSYS:	1259	case -ERESTARTSYS:
1260	case -EINTR:	1260	case -EINTR:
1261	return VM_FAULT_NOPAGE;	1261	return VM_FAULT_NOPAGE;
1262	case -ENOMEM:	1262	case -ENOMEM:
1263	return VM_FAULT_OOM;	1263	return VM_FAULT_OOM;
1264	default:	1264	default:
1265	return VM_FAULT_SIGBUS;	1265	return VM_FAULT_SIGBUS;
1266	}	1266	}
1267	}	1267	}
1268		1268
1269	/**	1269	/**
1270	* i915_gem_release_mmap - remove physical page mappings	1270	* i915_gem_release_mmap - remove physical page mappings
1271	* @obj: obj in question	1271	* @obj: obj in question
1272	*	1272	*
1273	* Preserve the reservation of the mmapping with the DRM core code, but	1273	* Preserve the reservation of the mmapping with the DRM core code, but
1274	* relinquish ownership of the pages back to the system.	1274	* relinquish ownership of the pages back to the system.
1275	*	1275	*
1276	* It is vital that we remove the page mapping if we have mapped a tiled	1276	* It is vital that we remove the page mapping if we have mapped a tiled
1277	* object through the GTT and then lose the fence register due to	1277	* object through the GTT and then lose the fence register due to
1278	* resource pressure. Similarly if the object has been moved out of the	1278	* resource pressure. Similarly if the object has been moved out of the
1279	* aperture, than pages mapped into userspace must be revoked. Removing the	1279	* aperture, than pages mapped into userspace must be revoked. Removing the
1280	* mapping will then trigger a page fault on the next user access, allowing	1280	* mapping will then trigger a page fault on the next user access, allowing
1281	* fixup by i915_gem_fault().	1281	* fixup by i915_gem_fault().
1282	*/	1282	*/
1283	void	1283	void
1284	i915_gem_release_mmap(struct drm_i915_gem_object *obj)	1284	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1285	{	1285	{
1286	if (!obj->fault_mappable)	1286	if (!obj->fault_mappable)
1287	return;	1287	return;
1288		1288
1289	if (obj->base.dev->dev_mapping)	1289	if (obj->base.dev->dev_mapping)
1290	unmap_mapping_range(obj->base.dev->dev_mapping,	1290	unmap_mapping_range(obj->base.dev->dev_mapping,
1291	(loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,	1291	(loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1292	obj->base.size, 1);	1292	obj->base.size, 1);
1293		1293
1294	obj->fault_mappable = false;	1294	obj->fault_mappable = false;
1295	}	1295	}
1296		1296
1297	static uint32_t	1297	static uint32_t
1298	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)	1298	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1299	{	1299	{
1300	uint32_t gtt_size;	1300	uint32_t gtt_size;
1301		1301
1302	if (INTEL_INFO(dev)->gen >= 4 \|\|	1302	if (INTEL_INFO(dev)->gen >= 4 \|\|
1303	tiling_mode == I915_TILING_NONE)	1303	tiling_mode == I915_TILING_NONE)
1304	return size;	1304	return size;
1305		1305
1306	/* Previous chips need a power-of-two fence region when tiling */	1306	/* Previous chips need a power-of-two fence region when tiling */
1307	if (INTEL_INFO(dev)->gen == 3)	1307	if (INTEL_INFO(dev)->gen == 3)
1308	gtt_size = 1024*1024;	1308	gtt_size = 1024*1024;
1309	else	1309	else
1310	gtt_size = 512*1024;	1310	gtt_size = 512*1024;
1311		1311
1312	while (gtt_size < size)	1312	while (gtt_size < size)
1313	gtt_size <<= 1;	1313	gtt_size <<= 1;
1314		1314
1315	return gtt_size;	1315	return gtt_size;
1316	}	1316	}
1317		1317
1318	/**	1318	/**
1319	* i915_gem_get_gtt_alignment - return required GTT alignment for an object	1319	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
1320	* @obj: object to check	1320	* @obj: object to check
1321	*	1321	*
1322	* Return the required GTT alignment for an object, taking into account	1322	* Return the required GTT alignment for an object, taking into account
1323	* potential fence register mapping.	1323	* potential fence register mapping.
1324	*/	1324	*/
1325	static uint32_t	1325	static uint32_t
1326	i915_gem_get_gtt_alignment(struct drm_device *dev,	1326	i915_gem_get_gtt_alignment(struct drm_device *dev,
1327	uint32_t size,	1327	uint32_t size,
1328	int tiling_mode)	1328	int tiling_mode)
1329	{	1329	{
1330	/*	1330	/*
1331	* Minimum alignment is 4k (GTT page size), but might be greater	1331	* Minimum alignment is 4k (GTT page size), but might be greater
1332	* if a fence register is needed for the object.	1332	* if a fence register is needed for the object.
1333	*/	1333	*/
1334	if (INTEL_INFO(dev)->gen >= 4 \|\|	1334	if (INTEL_INFO(dev)->gen >= 4 \|\|
1335	tiling_mode == I915_TILING_NONE)	1335	tiling_mode == I915_TILING_NONE)
1336	return 4096;	1336	return 4096;
1337		1337
1338	/*	1338	/*
1339	* Previous chips need to be aligned to the size of the smallest	1339	* Previous chips need to be aligned to the size of the smallest
1340	* fence register that can contain the object.	1340	* fence register that can contain the object.
1341	*/	1341	*/
1342	return i915_gem_get_gtt_size(dev, size, tiling_mode);	1342	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1343	}	1343	}
1344		1344
1345	/**	1345	/**
1346	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an	1346	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1347	* unfenced object	1347	* unfenced object
1348	* @dev: the device	1348	* @dev: the device
1349	* @size: size of the object	1349	* @size: size of the object
1350	* @tiling_mode: tiling mode of the object	1350	* @tiling_mode: tiling mode of the object
1351	*	1351	*
1352	* Return the required GTT alignment for an object, only taking into account	1352	* Return the required GTT alignment for an object, only taking into account
1353	* unfenced tiled surface requirements.	1353	* unfenced tiled surface requirements.
1354	*/	1354	*/
1355	uint32_t	1355	uint32_t
1356	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,	1356	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1357	uint32_t size,	1357	uint32_t size,
1358	int tiling_mode)	1358	int tiling_mode)
1359	{	1359	{
1360	/*	1360	/*
1361	* Minimum alignment is 4k (GTT page size) for sane hw.	1361	* Minimum alignment is 4k (GTT page size) for sane hw.
1362	*/	1362	*/
1363	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|	1363	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
1364	tiling_mode == I915_TILING_NONE)	1364	tiling_mode == I915_TILING_NONE)
1365	return 4096;	1365	return 4096;
1366		1366
1367	/* Previous hardware however needs to be aligned to a power-of-two	1367	/* Previous hardware however needs to be aligned to a power-of-two
1368	* tile height. The simplest method for determining this is to reuse	1368	* tile height. The simplest method for determining this is to reuse
1369	* the power-of-tile object size.	1369	* the power-of-tile object size.
1370	*/	1370	*/
1371	return i915_gem_get_gtt_size(dev, size, tiling_mode);	1371	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1372	}	1372	}
1373		1373
1374	int	1374	int
1375	i915_gem_mmap_gtt(struct drm_file *file,	1375	i915_gem_mmap_gtt(struct drm_file *file,
1376	struct drm_device *dev,	1376	struct drm_device *dev,
1377	uint32_t handle,	1377	uint32_t handle,
1378	uint64_t *offset)	1378	uint64_t *offset)
1379	{	1379	{
1380	struct drm_i915_private *dev_priv = dev->dev_private;	1380	struct drm_i915_private *dev_priv = dev->dev_private;
1381	struct drm_i915_gem_object *obj;	1381	struct drm_i915_gem_object *obj;
1382	int ret;	1382	int ret;
1383		1383
1384	if (!(dev->driver->driver_features & DRIVER_GEM))	1384	if (!(dev->driver->driver_features & DRIVER_GEM))
1385	return -ENODEV;	1385	return -ENODEV;
1386		1386
1387	ret = i915_mutex_lock_interruptible(dev);	1387	ret = i915_mutex_lock_interruptible(dev);
1388	if (ret)	1388	if (ret)
1389	return ret;	1389	return ret;
1390		1390
1391	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));	1391	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1392	if (&obj->base == NULL) {	1392	if (&obj->base == NULL) {
1393	ret = -ENOENT;	1393	ret = -ENOENT;
1394	goto unlock;	1394	goto unlock;
1395	}	1395	}
1396		1396
1397	if (obj->base.size > dev_priv->mm.gtt_mappable_end) {	1397	if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1398	ret = -E2BIG;	1398	ret = -E2BIG;
1399	goto out;	1399	goto out;
1400	}	1400	}
1401		1401
1402	if (obj->madv != I915_MADV_WILLNEED) {	1402	if (obj->madv != I915_MADV_WILLNEED) {
1403	DRM_ERROR("Attempting to mmap a purgeable buffer\n");	1403	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1404	ret = -EINVAL;	1404	ret = -EINVAL;
1405	goto out;	1405	goto out;
1406	}	1406	}
1407		1407
1408	if (!obj->base.map_list.map) {	1408	if (!obj->base.map_list.map) {
1409	ret = drm_gem_create_mmap_offset(&obj->base);	1409	ret = drm_gem_create_mmap_offset(&obj->base);
1410	if (ret)	1410	if (ret)
1411	goto out;	1411	goto out;
1412	}	1412	}
1413		1413
1414	*offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;	1414	*offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1415		1415
1416	out:	1416	out:
1417	drm_gem_object_unreference(&obj->base);	1417	drm_gem_object_unreference(&obj->base);
1418	unlock:	1418	unlock:
1419	mutex_unlock(&dev->struct_mutex);	1419	mutex_unlock(&dev->struct_mutex);
1420	return ret;	1420	return ret;
1421	}	1421	}
1422		1422
1423	/**	1423	/**
1424	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing	1424	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1425	* @dev: DRM device	1425	* @dev: DRM device
1426	* @data: GTT mapping ioctl data	1426	* @data: GTT mapping ioctl data
1427	* @file: GEM object info	1427	* @file: GEM object info
1428	*	1428	*
1429	* Simply returns the fake offset to userspace so it can mmap it.	1429	* Simply returns the fake offset to userspace so it can mmap it.
1430	* The mmap call will end up in drm_gem_mmap(), which will set things	1430	* The mmap call will end up in drm_gem_mmap(), which will set things
1431	* up so we can get faults in the handler above.	1431	* up so we can get faults in the handler above.
1432	*	1432	*
1433	* The fault handler will take care of binding the object into the GTT	1433	* The fault handler will take care of binding the object into the GTT
1434	* (since it may have been evicted to make room for something), allocating	1434	* (since it may have been evicted to make room for something), allocating
1435	* a fence register, and mapping the appropriate aperture address into	1435	* a fence register, and mapping the appropriate aperture address into
1436	* userspace.	1436	* userspace.
1437	*/	1437	*/
1438	int	1438	int
1439	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,	1439	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
1440	struct drm_file *file)	1440	struct drm_file *file)
1441	{	1441	{
1442	struct drm_i915_gem_mmap_gtt *args = data;	1442	struct drm_i915_gem_mmap_gtt *args = data;
1443		1443
1444	if (!(dev->driver->driver_features & DRIVER_GEM))	1444	if (!(dev->driver->driver_features & DRIVER_GEM))
1445	return -ENODEV;	1445	return -ENODEV;
1446		1446
1447	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);	1447	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1448	}	1448	}
1449		1449
1450		1450
1451	static int	1451	static int
1452	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,	1452	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1453	gfp_t gfpmask)	1453	gfp_t gfpmask)
1454	{	1454	{
1455	int page_count, i;	1455	int page_count, i;
1456	struct address_space *mapping;	1456	struct address_space *mapping;
1457	struct inode *inode;	1457	struct inode *inode;
1458	struct page *page;	1458	struct page *page;
1459		1459
1460	/* Get the list of pages out of our struct file. They'll be pinned	1460	/* Get the list of pages out of our struct file. They'll be pinned
1461	* at this point until we release them.	1461	* at this point until we release them.
1462	*/	1462	*/
1463	page_count = obj->base.size / PAGE_SIZE;	1463	page_count = obj->base.size / PAGE_SIZE;
1464	BUG_ON(obj->pages != NULL);	1464	BUG_ON(obj->pages != NULL);
1465	obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));	1465	obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1466	if (obj->pages == NULL)	1466	if (obj->pages == NULL)
1467	return -ENOMEM;	1467	return -ENOMEM;
1468		1468
1469	inode = obj->base.filp->f_path.dentry->d_inode;	1469	inode = obj->base.filp->f_path.dentry->d_inode;
1470	mapping = inode->i_mapping;	1470	mapping = inode->i_mapping;
1471	gfpmask \|= mapping_gfp_mask(mapping);	1471	gfpmask \|= mapping_gfp_mask(mapping);
1472		1472
1473	for (i = 0; i < page_count; i++) {	1473	for (i = 0; i < page_count; i++) {
1474	page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);	1474	page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1475	if (IS_ERR(page))	1475	if (IS_ERR(page))
1476	goto err_pages;	1476	goto err_pages;
1477		1477
1478	obj->pages[i] = page;	1478	obj->pages[i] = page;
1479	}	1479	}
1480		1480
1481	if (i915_gem_object_needs_bit17_swizzle(obj))	1481	if (i915_gem_object_needs_bit17_swizzle(obj))
1482	i915_gem_object_do_bit_17_swizzle(obj);	1482	i915_gem_object_do_bit_17_swizzle(obj);
1483		1483
1484	return 0;	1484	return 0;
1485		1485
1486	err_pages:	1486	err_pages:
1487	while (i--)	1487	while (i--)
1488	page_cache_release(obj->pages[i]);	1488	page_cache_release(obj->pages[i]);
1489		1489
1490	drm_free_large(obj->pages);	1490	drm_free_large(obj->pages);
1491	obj->pages = NULL;	1491	obj->pages = NULL;
1492	return PTR_ERR(page);	1492	return PTR_ERR(page);
1493	}	1493	}
1494		1494
1495	static void	1495	static void
1496	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)	1496	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1497	{	1497	{
1498	int page_count = obj->base.size / PAGE_SIZE;	1498	int page_count = obj->base.size / PAGE_SIZE;
1499	int i;	1499	int i;
1500		1500
1501	BUG_ON(obj->madv == __I915_MADV_PURGED);	1501	BUG_ON(obj->madv == __I915_MADV_PURGED);
1502		1502
1503	if (i915_gem_object_needs_bit17_swizzle(obj))	1503	if (i915_gem_object_needs_bit17_swizzle(obj))
1504	i915_gem_object_save_bit_17_swizzle(obj);	1504	i915_gem_object_save_bit_17_swizzle(obj);
1505		1505
1506	if (obj->madv == I915_MADV_DONTNEED)	1506	if (obj->madv == I915_MADV_DONTNEED)
1507	obj->dirty = 0;	1507	obj->dirty = 0;
1508		1508
1509	for (i = 0; i < page_count; i++) {	1509	for (i = 0; i < page_count; i++) {
1510	if (obj->dirty)	1510	if (obj->dirty)
1511	set_page_dirty(obj->pages[i]);	1511	set_page_dirty(obj->pages[i]);
1512		1512
1513	if (obj->madv == I915_MADV_WILLNEED)	1513	if (obj->madv == I915_MADV_WILLNEED)
1514	mark_page_accessed(obj->pages[i]);	1514	mark_page_accessed(obj->pages[i]);
1515		1515
1516	page_cache_release(obj->pages[i]);	1516	page_cache_release(obj->pages[i]);
1517	}	1517	}
1518	obj->dirty = 0;	1518	obj->dirty = 0;
1519		1519
1520	drm_free_large(obj->pages);	1520	drm_free_large(obj->pages);
1521	obj->pages = NULL;	1521	obj->pages = NULL;
1522	}	1522	}
1523		1523
1524	void	1524	void
1525	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,	1525	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1526	struct intel_ring_buffer *ring,	1526	struct intel_ring_buffer *ring,
1527	u32 seqno)	1527	u32 seqno)
1528	{	1528	{
1529	struct drm_device *dev = obj->base.dev;	1529	struct drm_device *dev = obj->base.dev;
1530	struct drm_i915_private *dev_priv = dev->dev_private;	1530	struct drm_i915_private *dev_priv = dev->dev_private;
1531		1531
1532	BUG_ON(ring == NULL);	1532	BUG_ON(ring == NULL);
1533	obj->ring = ring;	1533	obj->ring = ring;
1534		1534
1535	/* Add a reference if we're newly entering the active list. */	1535	/* Add a reference if we're newly entering the active list. */
1536	if (!obj->active) {	1536	if (!obj->active) {
1537	drm_gem_object_reference(&obj->base);	1537	drm_gem_object_reference(&obj->base);
1538	obj->active = 1;	1538	obj->active = 1;
1539	}	1539	}
1540		1540
1541	/* Move from whatever list we were on to the tail of execution. */	1541	/* Move from whatever list we were on to the tail of execution. */
1542	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);	1542	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1543	list_move_tail(&obj->ring_list, &ring->active_list);	1543	list_move_tail(&obj->ring_list, &ring->active_list);
1544		1544
1545	obj->last_rendering_seqno = seqno;	1545	obj->last_rendering_seqno = seqno;
1546	if (obj->fenced_gpu_access) {	1546	if (obj->fenced_gpu_access) {
1547	struct drm_i915_fence_reg *reg;	1547	struct drm_i915_fence_reg *reg;
1548		1548
1549	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);	1549	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1550		1550
1551	obj->last_fenced_seqno = seqno;	1551	obj->last_fenced_seqno = seqno;
1552	obj->last_fenced_ring = ring;	1552	obj->last_fenced_ring = ring;
1553		1553
1554	reg = &dev_priv->fence_regs[obj->fence_reg];	1554	reg = &dev_priv->fence_regs[obj->fence_reg];
1555	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);	1555	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1556	}	1556	}
1557	}	1557	}
1558		1558
1559	static void	1559	static void
1560	i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)	1560	i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1561	{	1561	{
1562	list_del_init(&obj->ring_list);	1562	list_del_init(&obj->ring_list);
1563	obj->last_rendering_seqno = 0;	1563	obj->last_rendering_seqno = 0;
1564	}	1564	}
1565		1565
1566	static void	1566	static void
1567	i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)	1567	i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1568	{	1568	{
1569	struct drm_device *dev = obj->base.dev;	1569	struct drm_device *dev = obj->base.dev;
1570	drm_i915_private_t *dev_priv = dev->dev_private;	1570	drm_i915_private_t *dev_priv = dev->dev_private;
1571		1571
1572	BUG_ON(!obj->active);	1572	BUG_ON(!obj->active);
1573	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);	1573	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1574		1574
1575	i915_gem_object_move_off_active(obj);	1575	i915_gem_object_move_off_active(obj);
1576	}	1576	}
1577		1577
1578	static void	1578	static void
1579	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)	1579	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1580	{	1580	{
1581	struct drm_device *dev = obj->base.dev;	1581	struct drm_device *dev = obj->base.dev;
1582	struct drm_i915_private *dev_priv = dev->dev_private;	1582	struct drm_i915_private *dev_priv = dev->dev_private;
1583		1583
1584	if (obj->pin_count != 0)	1584	if (obj->pin_count != 0)
1585	list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);	1585	list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1586	else	1586	else
1587	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);	1587	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1588		1588
1589	BUG_ON(!list_empty(&obj->gpu_write_list));	1589	BUG_ON(!list_empty(&obj->gpu_write_list));
1590	BUG_ON(!obj->active);	1590	BUG_ON(!obj->active);
1591	obj->ring = NULL;	1591	obj->ring = NULL;
1592		1592
1593	i915_gem_object_move_off_active(obj);	1593	i915_gem_object_move_off_active(obj);
1594	obj->fenced_gpu_access = false;	1594	obj->fenced_gpu_access = false;
1595		1595
1596	obj->active = 0;	1596	obj->active = 0;
1597	obj->pending_gpu_write = false;	1597	obj->pending_gpu_write = false;
1598	drm_gem_object_unreference(&obj->base);	1598	drm_gem_object_unreference(&obj->base);
1599		1599
1600	WARN_ON(i915_verify_lists(dev));	1600	WARN_ON(i915_verify_lists(dev));
1601	}	1601	}
1602		1602
1603	/* Immediately discard the backing storage */	1603	/* Immediately discard the backing storage */
1604	static void	1604	static void
1605	i915_gem_object_truncate(struct drm_i915_gem_object *obj)	1605	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1606	{	1606	{
1607	struct inode *inode;	1607	struct inode *inode;
1608		1608
1609	/* Our goal here is to return as much of the memory as	1609	/* Our goal here is to return as much of the memory as
1610	* is possible back to the system as we are called from OOM.	1610	* is possible back to the system as we are called from OOM.
1611	* To do this we must instruct the shmfs to drop all of its	1611	* To do this we must instruct the shmfs to drop all of its
1612	* backing pages, now.	1612	* backing pages, now.
1613	*/	1613	*/
1614	inode = obj->base.filp->f_path.dentry->d_inode;	1614	inode = obj->base.filp->f_path.dentry->d_inode;
1615	shmem_truncate_range(inode, 0, (loff_t)-1);	1615	shmem_truncate_range(inode, 0, (loff_t)-1);
1616		1616
1617	obj->madv = __I915_MADV_PURGED;	1617	obj->madv = __I915_MADV_PURGED;
1618	}	1618	}
1619		1619
1620	static inline int	1620	static inline int
1621	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)	1621	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1622	{	1622	{
1623	return obj->madv == I915_MADV_DONTNEED;	1623	return obj->madv == I915_MADV_DONTNEED;
1624	}	1624	}
1625		1625
1626	static void	1626	static void
1627	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,	1627	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1628	uint32_t flush_domains)	1628	uint32_t flush_domains)
1629	{	1629	{
1630	struct drm_i915_gem_object obj, next;	1630	struct drm_i915_gem_object obj, next;
1631		1631
1632	list_for_each_entry_safe(obj, next,	1632	list_for_each_entry_safe(obj, next,
1633	&ring->gpu_write_list,	1633	&ring->gpu_write_list,
1634	gpu_write_list) {	1634	gpu_write_list) {
1635	if (obj->base.write_domain & flush_domains) {	1635	if (obj->base.write_domain & flush_domains) {
1636	uint32_t old_write_domain = obj->base.write_domain;	1636	uint32_t old_write_domain = obj->base.write_domain;
1637		1637
1638	obj->base.write_domain = 0;	1638	obj->base.write_domain = 0;
1639	list_del_init(&obj->gpu_write_list);	1639	list_del_init(&obj->gpu_write_list);
1640	i915_gem_object_move_to_active(obj, ring,	1640	i915_gem_object_move_to_active(obj, ring,
1641	i915_gem_next_request_seqno(ring));	1641	i915_gem_next_request_seqno(ring));
1642		1642
1643	trace_i915_gem_object_change_domain(obj,	1643	trace_i915_gem_object_change_domain(obj,
1644	obj->base.read_domains,	1644	obj->base.read_domains,
1645	old_write_domain);	1645	old_write_domain);
1646	}	1646	}
1647	}	1647	}
1648	}	1648	}
1649		1649
1650	int	1650	int
1651	i915_add_request(struct intel_ring_buffer *ring,	1651	i915_add_request(struct intel_ring_buffer *ring,
1652	struct drm_file *file,	1652	struct drm_file *file,
1653	struct drm_i915_gem_request *request)	1653	struct drm_i915_gem_request *request)
1654	{	1654	{
1655	drm_i915_private_t *dev_priv = ring->dev->dev_private;	1655	drm_i915_private_t *dev_priv = ring->dev->dev_private;
1656	uint32_t seqno;	1656	uint32_t seqno;
1657	int was_empty;	1657	int was_empty;
1658	int ret;	1658	int ret;
1659		1659
1660	BUG_ON(request == NULL);	1660	BUG_ON(request == NULL);
1661		1661
1662	ret = ring->add_request(ring, &seqno);	1662	ret = ring->add_request(ring, &seqno);
1663	if (ret)	1663	if (ret)
1664	return ret;	1664	return ret;
1665		1665
1666	trace_i915_gem_request_add(ring, seqno);	1666	trace_i915_gem_request_add(ring, seqno);
1667		1667
1668	request->seqno = seqno;	1668	request->seqno = seqno;
1669	request->ring = ring;	1669	request->ring = ring;
1670	request->emitted_jiffies = jiffies;	1670	request->emitted_jiffies = jiffies;
1671	was_empty = list_empty(&ring->request_list);	1671	was_empty = list_empty(&ring->request_list);
1672	list_add_tail(&request->list, &ring->request_list);	1672	list_add_tail(&request->list, &ring->request_list);
1673		1673
1674	if (file) {	1674	if (file) {
1675	struct drm_i915_file_private *file_priv = file->driver_priv;	1675	struct drm_i915_file_private *file_priv = file->driver_priv;
1676		1676
1677	spin_lock(&file_priv->mm.lock);	1677	spin_lock(&file_priv->mm.lock);
1678	request->file_priv = file_priv;	1678	request->file_priv = file_priv;
1679	list_add_tail(&request->client_list,	1679	list_add_tail(&request->client_list,
1680	&file_priv->mm.request_list);	1680	&file_priv->mm.request_list);
1681	spin_unlock(&file_priv->mm.lock);	1681	spin_unlock(&file_priv->mm.lock);
1682	}	1682	}
1683		1683
1684	ring->outstanding_lazy_request = false;	1684	ring->outstanding_lazy_request = false;
1685		1685
1686	if (!dev_priv->mm.suspended) {	1686	if (!dev_priv->mm.suspended) {
1687	if (i915_enable_hangcheck) {	1687	if (i915_enable_hangcheck) {
1688	mod_timer(&dev_priv->hangcheck_timer,	1688	mod_timer(&dev_priv->hangcheck_timer,
1689	jiffies +	1689	jiffies +
1690	msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));	1690	msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1691	}	1691	}
1692	if (was_empty)	1692	if (was_empty)
1693	queue_delayed_work(dev_priv->wq,	1693	queue_delayed_work(dev_priv->wq,
1694	&dev_priv->mm.retire_work, HZ);	1694	&dev_priv->mm.retire_work, HZ);
1695	}	1695	}
1696	return 0;	1696	return 0;
1697	}	1697	}
1698		1698
1699	static inline void	1699	static inline void
1700	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)	1700	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1701	{	1701	{
1702	struct drm_i915_file_private *file_priv = request->file_priv;	1702	struct drm_i915_file_private *file_priv = request->file_priv;
1703		1703
1704	if (!file_priv)	1704	if (!file_priv)
1705	return;	1705	return;
1706		1706
1707	spin_lock(&file_priv->mm.lock);	1707	spin_lock(&file_priv->mm.lock);
1708	if (request->file_priv) {	1708	if (request->file_priv) {
1709	list_del(&request->client_list);	1709	list_del(&request->client_list);
1710	request->file_priv = NULL;	1710	request->file_priv = NULL;
1711	}	1711	}
1712	spin_unlock(&file_priv->mm.lock);	1712	spin_unlock(&file_priv->mm.lock);
1713	}	1713	}
1714		1714
1715	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,	1715	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1716	struct intel_ring_buffer *ring)	1716	struct intel_ring_buffer *ring)
1717	{	1717	{
1718	while (!list_empty(&ring->request_list)) {	1718	while (!list_empty(&ring->request_list)) {
1719	struct drm_i915_gem_request *request;	1719	struct drm_i915_gem_request *request;
1720		1720
1721	request = list_first_entry(&ring->request_list,	1721	request = list_first_entry(&ring->request_list,
1722	struct drm_i915_gem_request,	1722	struct drm_i915_gem_request,
1723	list);	1723	list);
1724		1724
1725	list_del(&request->list);	1725	list_del(&request->list);
1726	i915_gem_request_remove_from_client(request);	1726	i915_gem_request_remove_from_client(request);
1727	kfree(request);	1727	kfree(request);
1728	}	1728	}
1729		1729
1730	while (!list_empty(&ring->active_list)) {	1730	while (!list_empty(&ring->active_list)) {
1731	struct drm_i915_gem_object *obj;	1731	struct drm_i915_gem_object *obj;
1732		1732
1733	obj = list_first_entry(&ring->active_list,	1733	obj = list_first_entry(&ring->active_list,
1734	struct drm_i915_gem_object,	1734	struct drm_i915_gem_object,
1735	ring_list);	1735	ring_list);
1736		1736
1737	obj->base.write_domain = 0;	1737	obj->base.write_domain = 0;
1738	list_del_init(&obj->gpu_write_list);	1738	list_del_init(&obj->gpu_write_list);
1739	i915_gem_object_move_to_inactive(obj);	1739	i915_gem_object_move_to_inactive(obj);
1740	}	1740	}
1741	}	1741	}
1742		1742
1743	static void i915_gem_reset_fences(struct drm_device *dev)	1743	static void i915_gem_reset_fences(struct drm_device *dev)
1744	{	1744	{
1745	struct drm_i915_private *dev_priv = dev->dev_private;	1745	struct drm_i915_private *dev_priv = dev->dev_private;
1746	int i;	1746	int i;
1747		1747
1748	for (i = 0; i < dev_priv->num_fence_regs; i++) {	1748	for (i = 0; i < dev_priv->num_fence_regs; i++) {
1749	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];	1749	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1750	struct drm_i915_gem_object *obj = reg->obj;	1750	struct drm_i915_gem_object *obj = reg->obj;
1751		1751
1752	if (!obj)	1752	if (!obj)
1753	continue;	1753	continue;
1754		1754
1755	if (obj->tiling_mode)	1755	if (obj->tiling_mode)
1756	i915_gem_release_mmap(obj);	1756	i915_gem_release_mmap(obj);
1757		1757
1758	reg->obj->fence_reg = I915_FENCE_REG_NONE;	1758	reg->obj->fence_reg = I915_FENCE_REG_NONE;
1759	reg->obj->fenced_gpu_access = false;	1759	reg->obj->fenced_gpu_access = false;
1760	reg->obj->last_fenced_seqno = 0;	1760	reg->obj->last_fenced_seqno = 0;
1761	reg->obj->last_fenced_ring = NULL;	1761	reg->obj->last_fenced_ring = NULL;
1762	i915_gem_clear_fence_reg(dev, reg);	1762	i915_gem_clear_fence_reg(dev, reg);
1763	}	1763	}
1764	}	1764	}
1765		1765
1766	void i915_gem_reset(struct drm_device *dev)	1766	void i915_gem_reset(struct drm_device *dev)
1767	{	1767	{
1768	struct drm_i915_private *dev_priv = dev->dev_private;	1768	struct drm_i915_private *dev_priv = dev->dev_private;
1769	struct drm_i915_gem_object *obj;	1769	struct drm_i915_gem_object *obj;
1770	int i;	1770	int i;
1771		1771
1772	for (i = 0; i < I915_NUM_RINGS; i++)	1772	for (i = 0; i < I915_NUM_RINGS; i++)
1773	i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);	1773	i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1774		1774
1775	/* Remove anything from the flushing lists. The GPU cache is likely	1775	/* Remove anything from the flushing lists. The GPU cache is likely
1776	* to be lost on reset along with the data, so simply move the	1776	* to be lost on reset along with the data, so simply move the
1777	* lost bo to the inactive list.	1777	* lost bo to the inactive list.
1778	*/	1778	*/
1779	while (!list_empty(&dev_priv->mm.flushing_list)) {	1779	while (!list_empty(&dev_priv->mm.flushing_list)) {
1780	obj = list_first_entry(&dev_priv->mm.flushing_list,	1780	obj = list_first_entry(&dev_priv->mm.flushing_list,
1781	struct drm_i915_gem_object,	1781	struct drm_i915_gem_object,
1782	mm_list);	1782	mm_list);
1783		1783
1784	obj->base.write_domain = 0;	1784	obj->base.write_domain = 0;
1785	list_del_init(&obj->gpu_write_list);	1785	list_del_init(&obj->gpu_write_list);
1786	i915_gem_object_move_to_inactive(obj);	1786	i915_gem_object_move_to_inactive(obj);
1787	}	1787	}
1788		1788
1789	/* Move everything out of the GPU domains to ensure we do any	1789	/* Move everything out of the GPU domains to ensure we do any
1790	* necessary invalidation upon reuse.	1790	* necessary invalidation upon reuse.
1791	*/	1791	*/
1792	list_for_each_entry(obj,	1792	list_for_each_entry(obj,
1793	&dev_priv->mm.inactive_list,	1793	&dev_priv->mm.inactive_list,
1794	mm_list)	1794	mm_list)
1795	{	1795	{
1796	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;	1796	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1797	}	1797	}
1798		1798
1799	/* The fence registers are invalidated so clear them out */	1799	/* The fence registers are invalidated so clear them out */
1800	i915_gem_reset_fences(dev);	1800	i915_gem_reset_fences(dev);
1801	}	1801	}
1802		1802
1803	/**	1803	/**
1804	* This function clears the request list as sequence numbers are passed.	1804	* This function clears the request list as sequence numbers are passed.
1805	*/	1805	*/
1806	static void	1806	static void
1807	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)	1807	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1808	{	1808	{
1809	uint32_t seqno;	1809	uint32_t seqno;
1810	int i;	1810	int i;
1811		1811
1812	if (list_empty(&ring->request_list))	1812	if (list_empty(&ring->request_list))
1813	return;	1813	return;
1814		1814
1815	WARN_ON(i915_verify_lists(ring->dev));	1815	WARN_ON(i915_verify_lists(ring->dev));
1816		1816
1817	seqno = ring->get_seqno(ring);	1817	seqno = ring->get_seqno(ring);
1818		1818
1819	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)	1819	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1820	if (seqno >= ring->sync_seqno[i])	1820	if (seqno >= ring->sync_seqno[i])
1821	ring->sync_seqno[i] = 0;	1821	ring->sync_seqno[i] = 0;
1822		1822
1823	while (!list_empty(&ring->request_list)) {	1823	while (!list_empty(&ring->request_list)) {
1824	struct drm_i915_gem_request *request;	1824	struct drm_i915_gem_request *request;
1825		1825
1826	request = list_first_entry(&ring->request_list,	1826	request = list_first_entry(&ring->request_list,
1827	struct drm_i915_gem_request,	1827	struct drm_i915_gem_request,
1828	list);	1828	list);
1829		1829
1830	if (!i915_seqno_passed(seqno, request->seqno))	1830	if (!i915_seqno_passed(seqno, request->seqno))
1831	break;	1831	break;
1832		1832
1833	trace_i915_gem_request_retire(ring, request->seqno);	1833	trace_i915_gem_request_retire(ring, request->seqno);
1834		1834
1835	list_del(&request->list);	1835	list_del(&request->list);
1836	i915_gem_request_remove_from_client(request);	1836	i915_gem_request_remove_from_client(request);
1837	kfree(request);	1837	kfree(request);
1838	}	1838	}
1839		1839
1840	/* Move any buffers on the active list that are no longer referenced	1840	/* Move any buffers on the active list that are no longer referenced
1841	* by the ringbuffer to the flushing/inactive lists as appropriate.	1841	* by the ringbuffer to the flushing/inactive lists as appropriate.
1842	*/	1842	*/
1843	while (!list_empty(&ring->active_list)) {	1843	while (!list_empty(&ring->active_list)) {
1844	struct drm_i915_gem_object *obj;	1844	struct drm_i915_gem_object *obj;
1845		1845
1846	obj = list_first_entry(&ring->active_list,	1846	obj = list_first_entry(&ring->active_list,
1847	struct drm_i915_gem_object,	1847	struct drm_i915_gem_object,
1848	ring_list);	1848	ring_list);
1849		1849
1850	if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))	1850	if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1851	break;	1851	break;
1852		1852
1853	if (obj->base.write_domain != 0)	1853	if (obj->base.write_domain != 0)
1854	i915_gem_object_move_to_flushing(obj);	1854	i915_gem_object_move_to_flushing(obj);
1855	else	1855	else
1856	i915_gem_object_move_to_inactive(obj);	1856	i915_gem_object_move_to_inactive(obj);
1857	}	1857	}
1858		1858
1859	if (unlikely(ring->trace_irq_seqno &&	1859	if (unlikely(ring->trace_irq_seqno &&
1860	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {	1860	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1861	ring->irq_put(ring);	1861	ring->irq_put(ring);
1862	ring->trace_irq_seqno = 0;	1862	ring->trace_irq_seqno = 0;
1863	}	1863	}
1864		1864
1865	WARN_ON(i915_verify_lists(ring->dev));	1865	WARN_ON(i915_verify_lists(ring->dev));
1866	}	1866	}
1867		1867
1868	void	1868	void
1869	i915_gem_retire_requests(struct drm_device *dev)	1869	i915_gem_retire_requests(struct drm_device *dev)
1870	{	1870	{
1871	drm_i915_private_t *dev_priv = dev->dev_private;	1871	drm_i915_private_t *dev_priv = dev->dev_private;
1872	int i;	1872	int i;
1873		1873
1874	if (!list_empty(&dev_priv->mm.deferred_free_list)) {	1874	if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1875	struct drm_i915_gem_object obj, next;	1875	struct drm_i915_gem_object obj, next;
1876		1876
1877	/* We must be careful that during unbind() we do not	1877	/* We must be careful that during unbind() we do not
1878	* accidentally infinitely recurse into retire requests.	1878	* accidentally infinitely recurse into retire requests.
1879	* Currently:	1879	* Currently:
1880	* retire -> free -> unbind -> wait -> retire_ring	1880	* retire -> free -> unbind -> wait -> retire_ring
1881	*/	1881	*/
1882	list_for_each_entry_safe(obj, next,	1882	list_for_each_entry_safe(obj, next,
1883	&dev_priv->mm.deferred_free_list,	1883	&dev_priv->mm.deferred_free_list,
1884	mm_list)	1884	mm_list)
1885	i915_gem_free_object_tail(obj);	1885	i915_gem_free_object_tail(obj);
1886	}	1886	}
1887		1887
1888	for (i = 0; i < I915_NUM_RINGS; i++)	1888	for (i = 0; i < I915_NUM_RINGS; i++)
1889	i915_gem_retire_requests_ring(&dev_priv->ring[i]);	1889	i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1890	}	1890	}
1891		1891
1892	static void	1892	static void
1893	i915_gem_retire_work_handler(struct work_struct *work)	1893	i915_gem_retire_work_handler(struct work_struct *work)
1894	{	1894	{
1895	drm_i915_private_t *dev_priv;	1895	drm_i915_private_t *dev_priv;
1896	struct drm_device *dev;	1896	struct drm_device *dev;
1897	bool idle;	1897	bool idle;
1898	int i;	1898	int i;
1899		1899
1900	dev_priv = container_of(work, drm_i915_private_t,	1900	dev_priv = container_of(work, drm_i915_private_t,
1901	mm.retire_work.work);	1901	mm.retire_work.work);
1902	dev = dev_priv->dev;	1902	dev = dev_priv->dev;
1903		1903
1904	/* Come back later if the device is busy... */	1904	/* Come back later if the device is busy... */
1905	if (!mutex_trylock(&dev->struct_mutex)) {	1905	if (!mutex_trylock(&dev->struct_mutex)) {
1906	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);	1906	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1907	return;	1907	return;
1908	}	1908	}
1909		1909
1910	i915_gem_retire_requests(dev);	1910	i915_gem_retire_requests(dev);
1911		1911
1912	/* Send a periodic flush down the ring so we don't hold onto GEM	1912	/* Send a periodic flush down the ring so we don't hold onto GEM
1913	* objects indefinitely.	1913	* objects indefinitely.
1914	*/	1914	*/
1915	idle = true;	1915	idle = true;
1916	for (i = 0; i < I915_NUM_RINGS; i++) {	1916	for (i = 0; i < I915_NUM_RINGS; i++) {
1917	struct intel_ring_buffer *ring = &dev_priv->ring[i];	1917	struct intel_ring_buffer *ring = &dev_priv->ring[i];
1918		1918
1919	if (!list_empty(&ring->gpu_write_list)) {	1919	if (!list_empty(&ring->gpu_write_list)) {
1920	struct drm_i915_gem_request *request;	1920	struct drm_i915_gem_request *request;
1921	int ret;	1921	int ret;
1922		1922
1923	ret = i915_gem_flush_ring(ring,	1923	ret = i915_gem_flush_ring(ring,
1924	0, I915_GEM_GPU_DOMAINS);	1924	0, I915_GEM_GPU_DOMAINS);
1925	request = kzalloc(sizeof(*request), GFP_KERNEL);	1925	request = kzalloc(sizeof(*request), GFP_KERNEL);
1926	if (ret \|\| request == NULL \|\|	1926	if (ret \|\| request == NULL \|\|
1927	i915_add_request(ring, NULL, request))	1927	i915_add_request(ring, NULL, request))
1928	kfree(request);	1928	kfree(request);
1929	}	1929	}
1930		1930
1931	idle &= list_empty(&ring->request_list);	1931	idle &= list_empty(&ring->request_list);
1932	}	1932	}
1933		1933
1934	if (!dev_priv->mm.suspended && !idle)	1934	if (!dev_priv->mm.suspended && !idle)
1935	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);	1935	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1936		1936
1937	mutex_unlock(&dev->struct_mutex);	1937	mutex_unlock(&dev->struct_mutex);
1938	}	1938	}
1939		1939
1940	/**	1940	/**
1941	* Waits for a sequence number to be signaled, and cleans up the	1941	* Waits for a sequence number to be signaled, and cleans up the
1942	* request and object lists appropriately for that event.	1942	* request and object lists appropriately for that event.
1943	*/	1943	*/
1944	int	1944	int
1945	i915_wait_request(struct intel_ring_buffer *ring,	1945	i915_wait_request(struct intel_ring_buffer *ring,
1946	uint32_t seqno)	1946	uint32_t seqno)
1947	{	1947	{
1948	drm_i915_private_t *dev_priv = ring->dev->dev_private;	1948	drm_i915_private_t *dev_priv = ring->dev->dev_private;
1949	u32 ier;	1949	u32 ier;
1950	int ret = 0;	1950	int ret = 0;
1951		1951
1952	BUG_ON(seqno == 0);	1952	BUG_ON(seqno == 0);
1953		1953
1954	if (atomic_read(&dev_priv->mm.wedged)) {	1954	if (atomic_read(&dev_priv->mm.wedged)) {
1955	struct completion *x = &dev_priv->error_completion;	1955	struct completion *x = &dev_priv->error_completion;
1956	bool recovery_complete;	1956	bool recovery_complete;
1957	unsigned long flags;	1957	unsigned long flags;
1958		1958
1959	/* Give the error handler a chance to run. */	1959	/* Give the error handler a chance to run. */
1960	spin_lock_irqsave(&x->wait.lock, flags);	1960	spin_lock_irqsave(&x->wait.lock, flags);
1961	recovery_complete = x->done > 0;	1961	recovery_complete = x->done > 0;
1962	spin_unlock_irqrestore(&x->wait.lock, flags);	1962	spin_unlock_irqrestore(&x->wait.lock, flags);
1963		1963
1964	return recovery_complete ? -EIO : -EAGAIN;	1964	return recovery_complete ? -EIO : -EAGAIN;
1965	}	1965	}
1966		1966
1967	if (seqno == ring->outstanding_lazy_request) {	1967	if (seqno == ring->outstanding_lazy_request) {
1968	struct drm_i915_gem_request *request;	1968	struct drm_i915_gem_request *request;
1969		1969
1970	request = kzalloc(sizeof(*request), GFP_KERNEL);	1970	request = kzalloc(sizeof(*request), GFP_KERNEL);
1971	if (request == NULL)	1971	if (request == NULL)
1972	return -ENOMEM;	1972	return -ENOMEM;
1973		1973
1974	ret = i915_add_request(ring, NULL, request);	1974	ret = i915_add_request(ring, NULL, request);
1975	if (ret) {	1975	if (ret) {
1976	kfree(request);	1976	kfree(request);
1977	return ret;	1977	return ret;
1978	}	1978	}
1979		1979
1980	seqno = request->seqno;	1980	seqno = request->seqno;
1981	}	1981	}
1982		1982
1983	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {	1983	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
1984	if (HAS_PCH_SPLIT(ring->dev))	1984	if (HAS_PCH_SPLIT(ring->dev))
1985	ier = I915_READ(DEIER) \| I915_READ(GTIER);	1985	ier = I915_READ(DEIER) \| I915_READ(GTIER);
1986	else	1986	else
1987	ier = I915_READ(IER);	1987	ier = I915_READ(IER);
1988	if (!ier) {	1988	if (!ier) {
1989	DRM_ERROR("something (likely vbetool) disabled "	1989	DRM_ERROR("something (likely vbetool) disabled "
1990	"interrupts, re-enabling\n");	1990	"interrupts, re-enabling\n");
1991	ring->dev->driver->irq_preinstall(ring->dev);	1991	ring->dev->driver->irq_preinstall(ring->dev);
1992	ring->dev->driver->irq_postinstall(ring->dev);	1992	ring->dev->driver->irq_postinstall(ring->dev);
1993	}	1993	}
1994		1994
1995	trace_i915_gem_request_wait_begin(ring, seqno);	1995	trace_i915_gem_request_wait_begin(ring, seqno);
1996		1996
1997	ring->waiting_seqno = seqno;	1997	ring->waiting_seqno = seqno;
1998	if (ring->irq_get(ring)) {	1998	if (ring->irq_get(ring)) {
1999	if (dev_priv->mm.interruptible)	1999	if (dev_priv->mm.interruptible)
2000	ret = wait_event_interruptible(ring->irq_queue,	2000	ret = wait_event_interruptible(ring->irq_queue,
2001	i915_seqno_passed(ring->get_seqno(ring), seqno)	2001	i915_seqno_passed(ring->get_seqno(ring), seqno)
2002	\|\| atomic_read(&dev_priv->mm.wedged));	2002	\|\| atomic_read(&dev_priv->mm.wedged));
2003	else	2003	else
2004	wait_event(ring->irq_queue,	2004	wait_event(ring->irq_queue,
2005	i915_seqno_passed(ring->get_seqno(ring), seqno)	2005	i915_seqno_passed(ring->get_seqno(ring), seqno)
2006	\|\| atomic_read(&dev_priv->mm.wedged));	2006	\|\| atomic_read(&dev_priv->mm.wedged));
2007		2007
2008	ring->irq_put(ring);	2008	ring->irq_put(ring);
2009	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),	2009	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
2010	seqno) \|\|	2010	seqno) \|\|
2011	atomic_read(&dev_priv->mm.wedged), 3000))	2011	atomic_read(&dev_priv->mm.wedged), 3000))
2012	ret = -EBUSY;	2012	ret = -EBUSY;
2013	ring->waiting_seqno = 0;	2013	ring->waiting_seqno = 0;
2014		2014
2015	trace_i915_gem_request_wait_end(ring, seqno);	2015	trace_i915_gem_request_wait_end(ring, seqno);
2016	}	2016	}
2017	if (atomic_read(&dev_priv->mm.wedged))	2017	if (atomic_read(&dev_priv->mm.wedged))
2018	ret = -EAGAIN;	2018	ret = -EAGAIN;
2019		2019
2020	if (ret && ret != -ERESTARTSYS)	2020	if (ret && ret != -ERESTARTSYS)
2021	DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",	2021	DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2022	__func__, ret, seqno, ring->get_seqno(ring),	2022	__func__, ret, seqno, ring->get_seqno(ring),
2023	dev_priv->next_seqno);	2023	dev_priv->next_seqno);
2024		2024
2025	/* Directly dispatch request retiring. While we have the work queue	2025	/* Directly dispatch request retiring. While we have the work queue
2026	* to handle this, the waiter on a request often wants an associated	2026	* to handle this, the waiter on a request often wants an associated
2027	* buffer to have made it to the inactive list, and we would need	2027	* buffer to have made it to the inactive list, and we would need
2028	* a separate wait queue to handle that.	2028	* a separate wait queue to handle that.
2029	*/	2029	*/
2030	if (ret == 0)	2030	if (ret == 0)
2031	i915_gem_retire_requests_ring(ring);	2031	i915_gem_retire_requests_ring(ring);
2032		2032
2033	return ret;	2033	return ret;
2034	}	2034	}
2035		2035
2036	/**	2036	/**
2037	* Ensures that all rendering to the object has completed and the object is	2037	* Ensures that all rendering to the object has completed and the object is
2038	* safe to unbind from the GTT or access from the CPU.	2038	* safe to unbind from the GTT or access from the CPU.
2039	*/	2039	*/
2040	int	2040	int
2041	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)	2041	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2042	{	2042	{
2043	int ret;	2043	int ret;
2044		2044
2045	/* This function only exists to support waiting for existing rendering,	2045	/* This function only exists to support waiting for existing rendering,
2046	* not for emitting required flushes.	2046	* not for emitting required flushes.
2047	*/	2047	*/
2048	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);	2048	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2049		2049
2050	/* If there is rendering queued on the buffer being evicted, wait for	2050	/* If there is rendering queued on the buffer being evicted, wait for
2051	* it.	2051	* it.
2052	*/	2052	*/
2053	if (obj->active) {	2053	if (obj->active) {
2054	ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);	2054	ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2055	if (ret)	2055	if (ret)
2056	return ret;	2056	return ret;
2057	}	2057	}
2058		2058
2059	return 0;	2059	return 0;
2060	}	2060	}
2061		2061
2062	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)	2062	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2063	{	2063	{
2064	u32 old_write_domain, old_read_domains;	2064	u32 old_write_domain, old_read_domains;
2065		2065
2066	/* Act a barrier for all accesses through the GTT */	2066	/* Act a barrier for all accesses through the GTT */
2067	mb();	2067	mb();
2068		2068
2069	/* Force a pagefault for domain tracking on next user access */	2069	/* Force a pagefault for domain tracking on next user access */
2070	i915_gem_release_mmap(obj);	2070	i915_gem_release_mmap(obj);
2071		2071
2072	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)	2072	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2073	return;	2073	return;
2074		2074
2075	old_read_domains = obj->base.read_domains;	2075	old_read_domains = obj->base.read_domains;
2076	old_write_domain = obj->base.write_domain;	2076	old_write_domain = obj->base.write_domain;
2077		2077
2078	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;	2078	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2079	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;	2079	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2080		2080
2081	trace_i915_gem_object_change_domain(obj,	2081	trace_i915_gem_object_change_domain(obj,
2082	old_read_domains,	2082	old_read_domains,
2083	old_write_domain);	2083	old_write_domain);
2084	}	2084	}
2085		2085
2086	/**	2086	/**
2087	* Unbinds an object from the GTT aperture.	2087	* Unbinds an object from the GTT aperture.
2088	*/	2088	*/
2089	int	2089	int
2090	i915_gem_object_unbind(struct drm_i915_gem_object *obj)	2090	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2091	{	2091	{
2092	int ret = 0;	2092	int ret = 0;
2093		2093
2094	if (obj->gtt_space == NULL)	2094	if (obj->gtt_space == NULL)
2095	return 0;	2095	return 0;
2096		2096
2097	if (obj->pin_count != 0) {	2097	if (obj->pin_count != 0) {
2098	DRM_ERROR("Attempting to unbind pinned buffer\n");	2098	DRM_ERROR("Attempting to unbind pinned buffer\n");
2099	return -EINVAL;	2099	return -EINVAL;
2100	}	2100	}
2101		2101
2102	ret = i915_gem_object_finish_gpu(obj);	2102	ret = i915_gem_object_finish_gpu(obj);
2103	if (ret == -ERESTARTSYS)	2103	if (ret == -ERESTARTSYS)
2104	return ret;	2104	return ret;
2105	/* Continue on if we fail due to EIO, the GPU is hung so we	2105	/* Continue on if we fail due to EIO, the GPU is hung so we
2106	* should be safe and we need to cleanup or else we might	2106	* should be safe and we need to cleanup or else we might
2107	* cause memory corruption through use-after-free.	2107	* cause memory corruption through use-after-free.
2108	*/	2108	*/
2109		2109
2110	i915_gem_object_finish_gtt(obj);	2110	i915_gem_object_finish_gtt(obj);
2111		2111
2112	/* Move the object to the CPU domain to ensure that	2112	/* Move the object to the CPU domain to ensure that
2113	* any possible CPU writes while it's not in the GTT	2113	* any possible CPU writes while it's not in the GTT
2114	* are flushed when we go to remap it.	2114	* are flushed when we go to remap it.
2115	*/	2115	*/
2116	if (ret == 0)	2116	if (ret == 0)
2117	ret = i915_gem_object_set_to_cpu_domain(obj, 1);	2117	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2118	if (ret == -ERESTARTSYS)	2118	if (ret == -ERESTARTSYS)
2119	return ret;	2119	return ret;
2120	if (ret) {	2120	if (ret) {
2121	/* In the event of a disaster, abandon all caches and	2121	/* In the event of a disaster, abandon all caches and
2122	* hope for the best.	2122	* hope for the best.
2123	*/	2123	*/
2124	i915_gem_clflush_object(obj);	2124	i915_gem_clflush_object(obj);
2125	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;	2125	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2126	}	2126	}
2127		2127
2128	/* release the fence reg _after_ flushing */	2128	/* release the fence reg _after_ flushing */
2129	ret = i915_gem_object_put_fence(obj);	2129	ret = i915_gem_object_put_fence(obj);
2130	if (ret == -ERESTARTSYS)	2130	if (ret == -ERESTARTSYS)
2131	return ret;	2131	return ret;
2132		2132
2133	trace_i915_gem_object_unbind(obj);	2133	trace_i915_gem_object_unbind(obj);
2134		2134
2135	i915_gem_gtt_unbind_object(obj);	2135	i915_gem_gtt_unbind_object(obj);
2136	i915_gem_object_put_pages_gtt(obj);	2136	i915_gem_object_put_pages_gtt(obj);
2137		2137
2138	list_del_init(&obj->gtt_list);	2138	list_del_init(&obj->gtt_list);
2139	list_del_init(&obj->mm_list);	2139	list_del_init(&obj->mm_list);
2140	/* Avoid an unnecessary call to unbind on rebind. */	2140	/* Avoid an unnecessary call to unbind on rebind. */
2141	obj->map_and_fenceable = true;	2141	obj->map_and_fenceable = true;
2142		2142
2143	drm_mm_put_block(obj->gtt_space);	2143	drm_mm_put_block(obj->gtt_space);
2144	obj->gtt_space = NULL;	2144	obj->gtt_space = NULL;
2145	obj->gtt_offset = 0;	2145	obj->gtt_offset = 0;
2146		2146
2147	if (i915_gem_object_is_purgeable(obj))	2147	if (i915_gem_object_is_purgeable(obj))
2148	i915_gem_object_truncate(obj);	2148	i915_gem_object_truncate(obj);
2149		2149
2150	return ret;	2150	return ret;
2151	}	2151	}
2152		2152
2153	int	2153	int
2154	i915_gem_flush_ring(struct intel_ring_buffer *ring,	2154	i915_gem_flush_ring(struct intel_ring_buffer *ring,
2155	uint32_t invalidate_domains,	2155	uint32_t invalidate_domains,
2156	uint32_t flush_domains)	2156	uint32_t flush_domains)
2157	{	2157	{
2158	int ret;	2158	int ret;
2159		2159
2160	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)	2160	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2161	return 0;	2161	return 0;
2162		2162
2163	trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);	2163	trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2164		2164
2165	ret = ring->flush(ring, invalidate_domains, flush_domains);	2165	ret = ring->flush(ring, invalidate_domains, flush_domains);
2166	if (ret)	2166	if (ret)
2167	return ret;	2167	return ret;
2168		2168
2169	if (flush_domains & I915_GEM_GPU_DOMAINS)	2169	if (flush_domains & I915_GEM_GPU_DOMAINS)
2170	i915_gem_process_flushing_list(ring, flush_domains);	2170	i915_gem_process_flushing_list(ring, flush_domains);
2171		2171
2172	return 0;	2172	return 0;
2173	}	2173	}
2174		2174
2175	static int i915_ring_idle(struct intel_ring_buffer *ring)	2175	static int i915_ring_idle(struct intel_ring_buffer *ring)
2176	{	2176	{
2177	int ret;	2177	int ret;
2178		2178
2179	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))	2179	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2180	return 0;	2180	return 0;
2181		2181
2182	if (!list_empty(&ring->gpu_write_list)) {	2182	if (!list_empty(&ring->gpu_write_list)) {
2183	ret = i915_gem_flush_ring(ring,	2183	ret = i915_gem_flush_ring(ring,
2184	I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);	2184	I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2185	if (ret)	2185	if (ret)
2186	return ret;	2186	return ret;
2187	}	2187	}
2188		2188
2189	return i915_wait_request(ring, i915_gem_next_request_seqno(ring));	2189	return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2190	}	2190	}
2191		2191
2192	int	2192	int
2193	i915_gpu_idle(struct drm_device *dev)	2193	i915_gpu_idle(struct drm_device *dev)
2194	{	2194	{
2195	drm_i915_private_t *dev_priv = dev->dev_private;	2195	drm_i915_private_t *dev_priv = dev->dev_private;
2196	int ret, i;	2196	int ret, i;
2197		2197
2198	/* Flush everything onto the inactive list. */	2198	/* Flush everything onto the inactive list. */
2199	for (i = 0; i < I915_NUM_RINGS; i++) {	2199	for (i = 0; i < I915_NUM_RINGS; i++) {
2200	ret = i915_ring_idle(&dev_priv->ring[i]);	2200	ret = i915_ring_idle(&dev_priv->ring[i]);
2201	if (ret)	2201	if (ret)
2202	return ret;	2202	return ret;
2203	}	2203	}
2204		2204
2205	return 0;	2205	return 0;
2206	}	2206	}
2207		2207
2208	static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,	2208	static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2209	struct intel_ring_buffer *pipelined)	2209	struct intel_ring_buffer *pipelined)
2210	{	2210	{
2211	struct drm_device *dev = obj->base.dev;	2211	struct drm_device *dev = obj->base.dev;
2212	drm_i915_private_t *dev_priv = dev->dev_private;	2212	drm_i915_private_t *dev_priv = dev->dev_private;
2213	u32 size = obj->gtt_space->size;	2213	u32 size = obj->gtt_space->size;
2214	int regnum = obj->fence_reg;	2214	int regnum = obj->fence_reg;
2215	uint64_t val;	2215	uint64_t val;
2216		2216
2217	val = (uint64_t)((obj->gtt_offset + size - 4096) &	2217	val = (uint64_t)((obj->gtt_offset + size - 4096) &
2218	0xfffff000) << 32;	2218	0xfffff000) << 32;
2219	val \|= obj->gtt_offset & 0xfffff000;	2219	val \|= obj->gtt_offset & 0xfffff000;
2220	val \|= (uint64_t)((obj->stride / 128) - 1) <<	2220	val \|= (uint64_t)((obj->stride / 128) - 1) <<
2221	SANDYBRIDGE_FENCE_PITCH_SHIFT;	2221	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2222		2222
2223	if (obj->tiling_mode == I915_TILING_Y)	2223	if (obj->tiling_mode == I915_TILING_Y)
2224	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;	2224	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
2225	val \|= I965_FENCE_REG_VALID;	2225	val \|= I965_FENCE_REG_VALID;
2226		2226
2227	if (pipelined) {	2227	if (pipelined) {
2228	int ret = intel_ring_begin(pipelined, 6);	2228	int ret = intel_ring_begin(pipelined, 6);
2229	if (ret)	2229	if (ret)
2230	return ret;	2230	return ret;
2231		2231
2232	intel_ring_emit(pipelined, MI_NOOP);	2232	intel_ring_emit(pipelined, MI_NOOP);
2233	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));	2233	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2234	intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);	2234	intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2235	intel_ring_emit(pipelined, (u32)val);	2235	intel_ring_emit(pipelined, (u32)val);
2236	intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);	2236	intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2237	intel_ring_emit(pipelined, (u32)(val >> 32));	2237	intel_ring_emit(pipelined, (u32)(val >> 32));
2238	intel_ring_advance(pipelined);	2238	intel_ring_advance(pipelined);
2239	} else	2239	} else
2240	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);	2240	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2241		2241
2242	return 0;	2242	return 0;
2243	}	2243	}
2244		2244
2245	static int i965_write_fence_reg(struct drm_i915_gem_object *obj,	2245	static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2246	struct intel_ring_buffer *pipelined)	2246	struct intel_ring_buffer *pipelined)
2247	{	2247	{
2248	struct drm_device *dev = obj->base.dev;	2248	struct drm_device *dev = obj->base.dev;
2249	drm_i915_private_t *dev_priv = dev->dev_private;	2249	drm_i915_private_t *dev_priv = dev->dev_private;
2250	u32 size = obj->gtt_space->size;	2250	u32 size = obj->gtt_space->size;
2251	int regnum = obj->fence_reg;	2251	int regnum = obj->fence_reg;
2252	uint64_t val;	2252	uint64_t val;
2253		2253
2254	val = (uint64_t)((obj->gtt_offset + size - 4096) &	2254	val = (uint64_t)((obj->gtt_offset + size - 4096) &
2255	0xfffff000) << 32;	2255	0xfffff000) << 32;
2256	val \|= obj->gtt_offset & 0xfffff000;	2256	val \|= obj->gtt_offset & 0xfffff000;
2257	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;	2257	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2258	if (obj->tiling_mode == I915_TILING_Y)	2258	if (obj->tiling_mode == I915_TILING_Y)
2259	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;	2259	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
2260	val \|= I965_FENCE_REG_VALID;	2260	val \|= I965_FENCE_REG_VALID;
2261		2261
2262	if (pipelined) {	2262	if (pipelined) {
2263	int ret = intel_ring_begin(pipelined, 6);	2263	int ret = intel_ring_begin(pipelined, 6);
2264	if (ret)	2264	if (ret)
2265	return ret;	2265	return ret;
2266		2266
2267	intel_ring_emit(pipelined, MI_NOOP);	2267	intel_ring_emit(pipelined, MI_NOOP);
2268	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));	2268	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2269	intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);	2269	intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2270	intel_ring_emit(pipelined, (u32)val);	2270	intel_ring_emit(pipelined, (u32)val);
2271	intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);	2271	intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2272	intel_ring_emit(pipelined, (u32)(val >> 32));	2272	intel_ring_emit(pipelined, (u32)(val >> 32));
2273	intel_ring_advance(pipelined);	2273	intel_ring_advance(pipelined);
2274	} else	2274	} else
2275	I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);	2275	I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2276		2276
2277	return 0;	2277	return 0;
2278	}	2278	}
2279		2279
2280	static int i915_write_fence_reg(struct drm_i915_gem_object *obj,	2280	static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2281	struct intel_ring_buffer *pipelined)	2281	struct intel_ring_buffer *pipelined)
2282	{	2282	{
2283	struct drm_device *dev = obj->base.dev;	2283	struct drm_device *dev = obj->base.dev;
2284	drm_i915_private_t *dev_priv = dev->dev_private;	2284	drm_i915_private_t *dev_priv = dev->dev_private;
2285	u32 size = obj->gtt_space->size;	2285	u32 size = obj->gtt_space->size;
2286	u32 fence_reg, val, pitch_val;	2286	u32 fence_reg, val, pitch_val;
2287	int tile_width;	2287	int tile_width;
2288		2288
2289	if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|	2289	if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
2290	(size & -size) != size \|\|	2290	(size & -size) != size \|\|
2291	(obj->gtt_offset & (size - 1)),	2291	(obj->gtt_offset & (size - 1)),
2292	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",	2292	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2293	obj->gtt_offset, obj->map_and_fenceable, size))	2293	obj->gtt_offset, obj->map_and_fenceable, size))
2294	return -EINVAL;	2294	return -EINVAL;
2295		2295
2296	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))	2296	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2297	tile_width = 128;	2297	tile_width = 128;
2298	else	2298	else
2299	tile_width = 512;	2299	tile_width = 512;
2300		2300
2301	/* Note: pitch better be a power of two tile widths */	2301	/* Note: pitch better be a power of two tile widths */
2302	pitch_val = obj->stride / tile_width;	2302	pitch_val = obj->stride / tile_width;
2303	pitch_val = ffs(pitch_val) - 1;	2303	pitch_val = ffs(pitch_val) - 1;
2304		2304
2305	val = obj->gtt_offset;	2305	val = obj->gtt_offset;
2306	if (obj->tiling_mode == I915_TILING_Y)	2306	if (obj->tiling_mode == I915_TILING_Y)
2307	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;	2307	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
2308	val \|= I915_FENCE_SIZE_BITS(size);	2308	val \|= I915_FENCE_SIZE_BITS(size);
2309	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;	2309	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
2310	val \|= I830_FENCE_REG_VALID;	2310	val \|= I830_FENCE_REG_VALID;
2311		2311
2312	fence_reg = obj->fence_reg;	2312	fence_reg = obj->fence_reg;
2313	if (fence_reg < 8)	2313	if (fence_reg < 8)
2314	fence_reg = FENCE_REG_830_0 + fence_reg * 4;	2314	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2315	else	2315	else
2316	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;	2316	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2317		2317
2318	if (pipelined) {	2318	if (pipelined) {
2319	int ret = intel_ring_begin(pipelined, 4);	2319	int ret = intel_ring_begin(pipelined, 4);
2320	if (ret)	2320	if (ret)
2321	return ret;	2321	return ret;
2322		2322
2323	intel_ring_emit(pipelined, MI_NOOP);	2323	intel_ring_emit(pipelined, MI_NOOP);
2324	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));	2324	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2325	intel_ring_emit(pipelined, fence_reg);	2325	intel_ring_emit(pipelined, fence_reg);
2326	intel_ring_emit(pipelined, val);	2326	intel_ring_emit(pipelined, val);
2327	intel_ring_advance(pipelined);	2327	intel_ring_advance(pipelined);
2328	} else	2328	} else
2329	I915_WRITE(fence_reg, val);	2329	I915_WRITE(fence_reg, val);
2330		2330
2331	return 0;	2331	return 0;
2332	}	2332	}
2333		2333
2334	static int i830_write_fence_reg(struct drm_i915_gem_object *obj,	2334	static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2335	struct intel_ring_buffer *pipelined)	2335	struct intel_ring_buffer *pipelined)
2336	{	2336	{
2337	struct drm_device *dev = obj->base.dev;	2337	struct drm_device *dev = obj->base.dev;
2338	drm_i915_private_t *dev_priv = dev->dev_private;	2338	drm_i915_private_t *dev_priv = dev->dev_private;
2339	u32 size = obj->gtt_space->size;	2339	u32 size = obj->gtt_space->size;
2340	int regnum = obj->fence_reg;	2340	int regnum = obj->fence_reg;
2341	uint32_t val;	2341	uint32_t val;
2342	uint32_t pitch_val;	2342	uint32_t pitch_val;
2343		2343
2344	if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|	2344	if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
2345	(size & -size) != size \|\|	2345	(size & -size) != size \|\|
2346	(obj->gtt_offset & (size - 1)),	2346	(obj->gtt_offset & (size - 1)),
2347	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",	2347	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2348	obj->gtt_offset, size))	2348	obj->gtt_offset, size))
2349	return -EINVAL;	2349	return -EINVAL;
2350		2350
2351	pitch_val = obj->stride / 128;	2351	pitch_val = obj->stride / 128;
2352	pitch_val = ffs(pitch_val) - 1;	2352	pitch_val = ffs(pitch_val) - 1;
2353		2353
2354	val = obj->gtt_offset;	2354	val = obj->gtt_offset;
2355	if (obj->tiling_mode == I915_TILING_Y)	2355	if (obj->tiling_mode == I915_TILING_Y)
2356	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;	2356	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
2357	val \|= I830_FENCE_SIZE_BITS(size);	2357	val \|= I830_FENCE_SIZE_BITS(size);
2358	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;	2358	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
2359	val \|= I830_FENCE_REG_VALID;	2359	val \|= I830_FENCE_REG_VALID;
2360		2360
2361	if (pipelined) {	2361	if (pipelined) {
2362	int ret = intel_ring_begin(pipelined, 4);	2362	int ret = intel_ring_begin(pipelined, 4);
2363	if (ret)	2363	if (ret)
2364	return ret;	2364	return ret;
2365		2365
2366	intel_ring_emit(pipelined, MI_NOOP);	2366	intel_ring_emit(pipelined, MI_NOOP);
2367	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));	2367	intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2368	intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);	2368	intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2369	intel_ring_emit(pipelined, val);	2369	intel_ring_emit(pipelined, val);
2370	intel_ring_advance(pipelined);	2370	intel_ring_advance(pipelined);
2371	} else	2371	} else
2372	I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);	2372	I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2373		2373
2374	return 0;	2374	return 0;
2375	}	2375	}
2376		2376
2377	static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)	2377	static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2378	{	2378	{
2379	return i915_seqno_passed(ring->get_seqno(ring), seqno);	2379	return i915_seqno_passed(ring->get_seqno(ring), seqno);
2380	}	2380	}
2381		2381
2382	static int	2382	static int
2383	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,	2383	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2384	struct intel_ring_buffer *pipelined)	2384	struct intel_ring_buffer *pipelined)
2385	{	2385	{
2386	int ret;	2386	int ret;
2387		2387
2388	if (obj->fenced_gpu_access) {	2388	if (obj->fenced_gpu_access) {
2389	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {	2389	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2390	ret = i915_gem_flush_ring(obj->last_fenced_ring,	2390	ret = i915_gem_flush_ring(obj->last_fenced_ring,
2391	0, obj->base.write_domain);	2391	0, obj->base.write_domain);
2392	if (ret)	2392	if (ret)
2393	return ret;	2393	return ret;
2394	}	2394	}
2395		2395
2396	obj->fenced_gpu_access = false;	2396	obj->fenced_gpu_access = false;
2397	}	2397	}
2398		2398
2399	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {	2399	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2400	if (!ring_passed_seqno(obj->last_fenced_ring,	2400	if (!ring_passed_seqno(obj->last_fenced_ring,
2401	obj->last_fenced_seqno)) {	2401	obj->last_fenced_seqno)) {
2402	ret = i915_wait_request(obj->last_fenced_ring,	2402	ret = i915_wait_request(obj->last_fenced_ring,
2403	obj->last_fenced_seqno);	2403	obj->last_fenced_seqno);
2404	if (ret)	2404	if (ret)
2405	return ret;	2405	return ret;
2406	}	2406	}
2407		2407
2408	obj->last_fenced_seqno = 0;	2408	obj->last_fenced_seqno = 0;
2409	obj->last_fenced_ring = NULL;	2409	obj->last_fenced_ring = NULL;
2410	}	2410	}
2411		2411
2412	/* Ensure that all CPU reads are completed before installing a fence	2412	/* Ensure that all CPU reads are completed before installing a fence
2413	* and all writes before removing the fence.	2413	* and all writes before removing the fence.
2414	*/	2414	*/
2415	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)	2415	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2416	mb();	2416	mb();
2417		2417
2418	return 0;	2418	return 0;
2419	}	2419	}
2420		2420
2421	int	2421	int
2422	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)	2422	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2423	{	2423	{
2424	int ret;	2424	int ret;
2425		2425
2426	if (obj->tiling_mode)	2426	if (obj->tiling_mode)
2427	i915_gem_release_mmap(obj);	2427	i915_gem_release_mmap(obj);
2428		2428
2429	ret = i915_gem_object_flush_fence(obj, NULL);	2429	ret = i915_gem_object_flush_fence(obj, NULL);
2430	if (ret)	2430	if (ret)
2431	return ret;	2431	return ret;
2432		2432
2433	if (obj->fence_reg != I915_FENCE_REG_NONE) {	2433	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2434	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;	2434	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2435	i915_gem_clear_fence_reg(obj->base.dev,	2435	i915_gem_clear_fence_reg(obj->base.dev,
2436	&dev_priv->fence_regs[obj->fence_reg]);	2436	&dev_priv->fence_regs[obj->fence_reg]);
2437		2437
2438	obj->fence_reg = I915_FENCE_REG_NONE;	2438	obj->fence_reg = I915_FENCE_REG_NONE;
2439	}	2439	}
2440		2440
2441	return 0;	2441	return 0;
2442	}	2442	}
2443		2443
2444	static struct drm_i915_fence_reg *	2444	static struct drm_i915_fence_reg *
2445	i915_find_fence_reg(struct drm_device *dev,	2445	i915_find_fence_reg(struct drm_device *dev,
2446	struct intel_ring_buffer *pipelined)	2446	struct intel_ring_buffer *pipelined)
2447	{	2447	{
2448	struct drm_i915_private *dev_priv = dev->dev_private;	2448	struct drm_i915_private *dev_priv = dev->dev_private;
2449	struct drm_i915_fence_reg reg, first, *avail;	2449	struct drm_i915_fence_reg reg, first, *avail;
2450	int i;	2450	int i;
2451		2451
2452	/* First try to find a free reg */	2452	/* First try to find a free reg */
2453	avail = NULL;	2453	avail = NULL;
2454	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {	2454	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2455	reg = &dev_priv->fence_regs[i];	2455	reg = &dev_priv->fence_regs[i];
2456	if (!reg->obj)	2456	if (!reg->obj)
2457	return reg;	2457	return reg;
2458		2458
2459	if (!reg->obj->pin_count)	2459	if (!reg->obj->pin_count)
2460	avail = reg;	2460	avail = reg;
2461	}	2461	}
2462		2462
2463	if (avail == NULL)	2463	if (avail == NULL)
2464	return NULL;	2464	return NULL;
2465		2465
2466	/* None available, try to steal one or wait for a user to finish */	2466	/* None available, try to steal one or wait for a user to finish */
2467	avail = first = NULL;	2467	avail = first = NULL;
2468	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {	2468	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2469	if (reg->obj->pin_count)	2469	if (reg->obj->pin_count)
2470	continue;	2470	continue;
2471		2471
2472	if (first == NULL)	2472	if (first == NULL)
2473	first = reg;	2473	first = reg;
2474		2474
2475	if (!pipelined \|\|	2475	if (!pipelined \|\|
2476	!reg->obj->last_fenced_ring \|\|	2476	!reg->obj->last_fenced_ring \|\|
2477	reg->obj->last_fenced_ring == pipelined) {	2477	reg->obj->last_fenced_ring == pipelined) {
2478	avail = reg;	2478	avail = reg;
2479	break;	2479	break;
2480	}	2480	}
2481	}	2481	}
2482		2482
2483	if (avail == NULL)	2483	if (avail == NULL)
2484	avail = first;	2484	avail = first;
2485		2485
2486	return avail;	2486	return avail;
2487	}	2487	}
2488		2488
2489	/**	2489	/**
2490	* i915_gem_object_get_fence - set up a fence reg for an object	2490	* i915_gem_object_get_fence - set up a fence reg for an object
2491	* @obj: object to map through a fence reg	2491	* @obj: object to map through a fence reg
2492	* @pipelined: ring on which to queue the change, or NULL for CPU access	2492	* @pipelined: ring on which to queue the change, or NULL for CPU access
2493	* @interruptible: must we wait uninterruptibly for the register to retire?	2493	* @interruptible: must we wait uninterruptibly for the register to retire?
2494	*	2494	*
2495	* When mapping objects through the GTT, userspace wants to be able to write	2495	* When mapping objects through the GTT, userspace wants to be able to write
2496	* to them without having to worry about swizzling if the object is tiled.	2496	* to them without having to worry about swizzling if the object is tiled.
2497	*	2497	*
2498	* This function walks the fence regs looking for a free one for @obj,	2498	* This function walks the fence regs looking for a free one for @obj,
2499	* stealing one if it can't find any.	2499	* stealing one if it can't find any.
2500	*	2500	*
2501	* It then sets up the reg based on the object's properties: address, pitch	2501	* It then sets up the reg based on the object's properties: address, pitch
2502	* and tiling format.	2502	* and tiling format.
2503	*/	2503	*/
2504	int	2504	int
2505	i915_gem_object_get_fence(struct drm_i915_gem_object *obj,	2505	i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2506	struct intel_ring_buffer *pipelined)	2506	struct intel_ring_buffer *pipelined)
2507	{	2507	{
2508	struct drm_device *dev = obj->base.dev;	2508	struct drm_device *dev = obj->base.dev;
2509	struct drm_i915_private *dev_priv = dev->dev_private;	2509	struct drm_i915_private *dev_priv = dev->dev_private;
2510	struct drm_i915_fence_reg *reg;	2510	struct drm_i915_fence_reg *reg;
2511	int ret;	2511	int ret;
2512		2512
2513	/* XXX disable pipelining. There are bugs. Shocking. */	2513	/* XXX disable pipelining. There are bugs. Shocking. */
2514	pipelined = NULL;	2514	pipelined = NULL;
2515		2515
2516	/* Just update our place in the LRU if our fence is getting reused. */	2516	/* Just update our place in the LRU if our fence is getting reused. */
2517	if (obj->fence_reg != I915_FENCE_REG_NONE) {	2517	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2518	reg = &dev_priv->fence_regs[obj->fence_reg];	2518	reg = &dev_priv->fence_regs[obj->fence_reg];
2519	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);	2519	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2520		2520
2521	if (obj->tiling_changed) {	2521	if (obj->tiling_changed) {
2522	ret = i915_gem_object_flush_fence(obj, pipelined);	2522	ret = i915_gem_object_flush_fence(obj, pipelined);
2523	if (ret)	2523	if (ret)
2524	return ret;	2524	return ret;
2525		2525
2526	if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)	2526	if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2527	pipelined = NULL;	2527	pipelined = NULL;
2528		2528
2529	if (pipelined) {	2529	if (pipelined) {
2530	reg->setup_seqno =	2530	reg->setup_seqno =
2531	i915_gem_next_request_seqno(pipelined);	2531	i915_gem_next_request_seqno(pipelined);
2532	obj->last_fenced_seqno = reg->setup_seqno;	2532	obj->last_fenced_seqno = reg->setup_seqno;
2533	obj->last_fenced_ring = pipelined;	2533	obj->last_fenced_ring = pipelined;
2534	}	2534	}
2535		2535
2536	goto update;	2536	goto update;
2537	}	2537	}
2538		2538
2539	if (!pipelined) {	2539	if (!pipelined) {
2540	if (reg->setup_seqno) {	2540	if (reg->setup_seqno) {
2541	if (!ring_passed_seqno(obj->last_fenced_ring,	2541	if (!ring_passed_seqno(obj->last_fenced_ring,
2542	reg->setup_seqno)) {	2542	reg->setup_seqno)) {
2543	ret = i915_wait_request(obj->last_fenced_ring,	2543	ret = i915_wait_request(obj->last_fenced_ring,
2544	reg->setup_seqno);	2544	reg->setup_seqno);
2545	if (ret)	2545	if (ret)
2546	return ret;	2546	return ret;
2547	}	2547	}
2548		2548
2549	reg->setup_seqno = 0;	2549	reg->setup_seqno = 0;
2550	}	2550	}
2551	} else if (obj->last_fenced_ring &&	2551	} else if (obj->last_fenced_ring &&
2552	obj->last_fenced_ring != pipelined) {	2552	obj->last_fenced_ring != pipelined) {
2553	ret = i915_gem_object_flush_fence(obj, pipelined);	2553	ret = i915_gem_object_flush_fence(obj, pipelined);
2554	if (ret)	2554	if (ret)
2555	return ret;	2555	return ret;
2556	}	2556	}
2557		2557
2558	return 0;	2558	return 0;
2559	}	2559	}
2560		2560
2561	reg = i915_find_fence_reg(dev, pipelined);	2561	reg = i915_find_fence_reg(dev, pipelined);
2562	if (reg == NULL)	2562	if (reg == NULL)
2563	return -ENOSPC;	2563	return -ENOSPC;
2564		2564
2565	ret = i915_gem_object_flush_fence(obj, pipelined);	2565	ret = i915_gem_object_flush_fence(obj, pipelined);
2566	if (ret)	2566	if (ret)
2567	return ret;	2567	return ret;
2568		2568
2569	if (reg->obj) {	2569	if (reg->obj) {
2570	struct drm_i915_gem_object *old = reg->obj;	2570	struct drm_i915_gem_object *old = reg->obj;
2571		2571
2572	drm_gem_object_reference(&old->base);	2572	drm_gem_object_reference(&old->base);
2573		2573
2574	if (old->tiling_mode)	2574	if (old->tiling_mode)
2575	i915_gem_release_mmap(old);	2575	i915_gem_release_mmap(old);
2576		2576
2577	ret = i915_gem_object_flush_fence(old, pipelined);	2577	ret = i915_gem_object_flush_fence(old, pipelined);
2578	if (ret) {	2578	if (ret) {
2579	drm_gem_object_unreference(&old->base);	2579	drm_gem_object_unreference(&old->base);
2580	return ret;	2580	return ret;
2581	}	2581	}
2582		2582
2583	if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)	2583	if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2584	pipelined = NULL;	2584	pipelined = NULL;
2585		2585
2586	old->fence_reg = I915_FENCE_REG_NONE;	2586	old->fence_reg = I915_FENCE_REG_NONE;
2587	old->last_fenced_ring = pipelined;	2587	old->last_fenced_ring = pipelined;
2588	old->last_fenced_seqno =	2588	old->last_fenced_seqno =
2589	pipelined ? i915_gem_next_request_seqno(pipelined) : 0;	2589	pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2590		2590
2591	drm_gem_object_unreference(&old->base);	2591	drm_gem_object_unreference(&old->base);
2592	} else if (obj->last_fenced_seqno == 0)	2592	} else if (obj->last_fenced_seqno == 0)
2593	pipelined = NULL;	2593	pipelined = NULL;
2594		2594
2595	reg->obj = obj;	2595	reg->obj = obj;
2596	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);	2596	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2597	obj->fence_reg = reg - dev_priv->fence_regs;	2597	obj->fence_reg = reg - dev_priv->fence_regs;
2598	obj->last_fenced_ring = pipelined;	2598	obj->last_fenced_ring = pipelined;
2599		2599
2600	reg->setup_seqno =	2600	reg->setup_seqno =
2601	pipelined ? i915_gem_next_request_seqno(pipelined) : 0;	2601	pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2602	obj->last_fenced_seqno = reg->setup_seqno;	2602	obj->last_fenced_seqno = reg->setup_seqno;
2603		2603
2604	update:	2604	update:
2605	obj->tiling_changed = false;	2605	obj->tiling_changed = false;
2606	switch (INTEL_INFO(dev)->gen) {	2606	switch (INTEL_INFO(dev)->gen) {
2607	case 7:	2607	case 7:
2608	case 6:	2608	case 6:
2609	ret = sandybridge_write_fence_reg(obj, pipelined);	2609	ret = sandybridge_write_fence_reg(obj, pipelined);
2610	break;	2610	break;
2611	case 5:	2611	case 5:
2612	case 4:	2612	case 4:
2613	ret = i965_write_fence_reg(obj, pipelined);	2613	ret = i965_write_fence_reg(obj, pipelined);
2614	break;	2614	break;
2615	case 3:	2615	case 3:
2616	ret = i915_write_fence_reg(obj, pipelined);	2616	ret = i915_write_fence_reg(obj, pipelined);
2617	break;	2617	break;
2618	case 2:	2618	case 2:
2619	ret = i830_write_fence_reg(obj, pipelined);	2619	ret = i830_write_fence_reg(obj, pipelined);
2620	break;	2620	break;
2621	}	2621	}
2622		2622
2623	return ret;	2623	return ret;
2624	}	2624	}
2625		2625
2626	/**	2626	/**
2627	* i915_gem_clear_fence_reg - clear out fence register info	2627	* i915_gem_clear_fence_reg - clear out fence register info
2628	* @obj: object to clear	2628	* @obj: object to clear
2629	*	2629	*
2630	* Zeroes out the fence register itself and clears out the associated	2630	* Zeroes out the fence register itself and clears out the associated
2631	* data structures in dev_priv and obj.	2631	* data structures in dev_priv and obj.
2632	*/	2632	*/
2633	static void	2633	static void
2634	i915_gem_clear_fence_reg(struct drm_device *dev,	2634	i915_gem_clear_fence_reg(struct drm_device *dev,
2635	struct drm_i915_fence_reg *reg)	2635	struct drm_i915_fence_reg *reg)
2636	{	2636	{
2637	drm_i915_private_t *dev_priv = dev->dev_private;	2637	drm_i915_private_t *dev_priv = dev->dev_private;
2638	uint32_t fence_reg = reg - dev_priv->fence_regs;	2638	uint32_t fence_reg = reg - dev_priv->fence_regs;
2639		2639
2640	switch (INTEL_INFO(dev)->gen) {	2640	switch (INTEL_INFO(dev)->gen) {
2641	case 7:	2641	case 7:
2642	case 6:	2642	case 6:
2643	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);	2643	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2644	break;	2644	break;
2645	case 5:	2645	case 5:
2646	case 4:	2646	case 4:
2647	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);	2647	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2648	break;	2648	break;
2649	case 3:	2649	case 3:
2650	if (fence_reg >= 8)	2650	if (fence_reg >= 8)
2651	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;	2651	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2652	else	2652	else
2653	case 2:	2653	case 2:
2654	fence_reg = FENCE_REG_830_0 + fence_reg * 4;	2654	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2655		2655
2656	I915_WRITE(fence_reg, 0);	2656	I915_WRITE(fence_reg, 0);
2657	break;	2657	break;
2658	}	2658	}
2659		2659
2660	list_del_init(&reg->lru_list);	2660	list_del_init(&reg->lru_list);
2661	reg->obj = NULL;	2661	reg->obj = NULL;
2662	reg->setup_seqno = 0;	2662	reg->setup_seqno = 0;
2663	}	2663	}
2664		2664
2665	/**	2665	/**
2666	* Finds free space in the GTT aperture and binds the object there.	2666	* Finds free space in the GTT aperture and binds the object there.
2667	*/	2667	*/
2668	static int	2668	static int
2669	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,	2669	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2670	unsigned alignment,	2670	unsigned alignment,
2671	bool map_and_fenceable)	2671	bool map_and_fenceable)
2672	{	2672	{
2673	struct drm_device *dev = obj->base.dev;	2673	struct drm_device *dev = obj->base.dev;
2674	drm_i915_private_t *dev_priv = dev->dev_private;	2674	drm_i915_private_t *dev_priv = dev->dev_private;
2675	struct drm_mm_node *free_space;	2675	struct drm_mm_node *free_space;
2676	gfp_t gfpmask = __GFP_NORETRY \| __GFP_NOWARN;	2676	gfp_t gfpmask = __GFP_NORETRY \| __GFP_NOWARN;
2677	u32 size, fence_size, fence_alignment, unfenced_alignment;	2677	u32 size, fence_size, fence_alignment, unfenced_alignment;
2678	bool mappable, fenceable;	2678	bool mappable, fenceable;
2679	int ret;	2679	int ret;
2680		2680
2681	if (obj->madv != I915_MADV_WILLNEED) {	2681	if (obj->madv != I915_MADV_WILLNEED) {
2682	DRM_ERROR("Attempting to bind a purgeable object\n");	2682	DRM_ERROR("Attempting to bind a purgeable object\n");
2683	return -EINVAL;	2683	return -EINVAL;
2684	}	2684	}
2685		2685
2686	fence_size = i915_gem_get_gtt_size(dev,	2686	fence_size = i915_gem_get_gtt_size(dev,
2687	obj->base.size,	2687	obj->base.size,
2688	obj->tiling_mode);	2688	obj->tiling_mode);
2689	fence_alignment = i915_gem_get_gtt_alignment(dev,	2689	fence_alignment = i915_gem_get_gtt_alignment(dev,
2690	obj->base.size,	2690	obj->base.size,
2691	obj->tiling_mode);	2691	obj->tiling_mode);
2692	unfenced_alignment =	2692	unfenced_alignment =
2693	i915_gem_get_unfenced_gtt_alignment(dev,	2693	i915_gem_get_unfenced_gtt_alignment(dev,
2694	obj->base.size,	2694	obj->base.size,
2695	obj->tiling_mode);	2695	obj->tiling_mode);
2696		2696
2697	if (alignment == 0)	2697	if (alignment == 0)
2698	alignment = map_and_fenceable ? fence_alignment :	2698	alignment = map_and_fenceable ? fence_alignment :
2699	unfenced_alignment;	2699	unfenced_alignment;
2700	if (map_and_fenceable && alignment & (fence_alignment - 1)) {	2700	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2701	DRM_ERROR("Invalid object alignment requested %u\n", alignment);	2701	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2702	return -EINVAL;	2702	return -EINVAL;
2703	}	2703	}
2704		2704
2705	size = map_and_fenceable ? fence_size : obj->base.size;	2705	size = map_and_fenceable ? fence_size : obj->base.size;
2706		2706
2707	/* If the object is bigger than the entire aperture, reject it early	2707	/* If the object is bigger than the entire aperture, reject it early
2708	* before evicting everything in a vain attempt to find space.	2708	* before evicting everything in a vain attempt to find space.
2709	*/	2709	*/
2710	if (obj->base.size >	2710	if (obj->base.size >
2711	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {	2711	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2712	DRM_ERROR("Attempting to bind an object larger than the aperture\n");	2712	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2713	return -E2BIG;	2713	return -E2BIG;
2714	}	2714	}
2715		2715
2716	search_free:	2716	search_free:
2717	if (map_and_fenceable)	2717	if (map_and_fenceable)
2718	free_space =	2718	free_space =
2719	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,	2719	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2720	size, alignment, 0,	2720	size, alignment, 0,
2721	dev_priv->mm.gtt_mappable_end,	2721	dev_priv->mm.gtt_mappable_end,
2722	0);	2722	0);
2723	else	2723	else
2724	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,	2724	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2725	size, alignment, 0);	2725	size, alignment, 0);
2726		2726
2727	if (free_space != NULL) {	2727	if (free_space != NULL) {
2728	if (map_and_fenceable)	2728	if (map_and_fenceable)
2729	obj->gtt_space =	2729	obj->gtt_space =
2730	drm_mm_get_block_range_generic(free_space,	2730	drm_mm_get_block_range_generic(free_space,
2731	size, alignment, 0,	2731	size, alignment, 0,
2732	dev_priv->mm.gtt_mappable_end,	2732	dev_priv->mm.gtt_mappable_end,
2733	0);	2733	0);
2734	else	2734	else
2735	obj->gtt_space =	2735	obj->gtt_space =
2736	drm_mm_get_block(free_space, size, alignment);	2736	drm_mm_get_block(free_space, size, alignment);
2737	}	2737	}
2738	if (obj->gtt_space == NULL) {	2738	if (obj->gtt_space == NULL) {
2739	/* If the gtt is empty and we're still having trouble	2739	/* If the gtt is empty and we're still having trouble
2740	* fitting our object in, we're out of memory.	2740	* fitting our object in, we're out of memory.
2741	*/	2741	*/
2742	ret = i915_gem_evict_something(dev, size, alignment,	2742	ret = i915_gem_evict_something(dev, size, alignment,
2743	map_and_fenceable);	2743	map_and_fenceable);
2744	if (ret)	2744	if (ret)
2745	return ret;	2745	return ret;
2746		2746
2747	goto search_free;	2747	goto search_free;
2748	}	2748	}
2749		2749
2750	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);	2750	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2751	if (ret) {	2751	if (ret) {
2752	drm_mm_put_block(obj->gtt_space);	2752	drm_mm_put_block(obj->gtt_space);
2753	obj->gtt_space = NULL;	2753	obj->gtt_space = NULL;
2754		2754
2755	if (ret == -ENOMEM) {	2755	if (ret == -ENOMEM) {
2756	/* first try to reclaim some memory by clearing the GTT */	2756	/* first try to reclaim some memory by clearing the GTT */
2757	ret = i915_gem_evict_everything(dev, false);	2757	ret = i915_gem_evict_everything(dev, false);
2758	if (ret) {	2758	if (ret) {
2759	/* now try to shrink everyone else */	2759	/* now try to shrink everyone else */
2760	if (gfpmask) {	2760	if (gfpmask) {
2761	gfpmask = 0;	2761	gfpmask = 0;
2762	goto search_free;	2762	goto search_free;
2763	}	2763	}
2764		2764
2765	return -ENOMEM;	2765	return -ENOMEM;
2766	}	2766	}
2767		2767
2768	goto search_free;	2768	goto search_free;
2769	}	2769	}
2770		2770
2771	return ret;	2771	return ret;
2772	}	2772	}
2773		2773
2774	ret = i915_gem_gtt_bind_object(obj);	2774	ret = i915_gem_gtt_bind_object(obj);
2775	if (ret) {	2775	if (ret) {
2776	i915_gem_object_put_pages_gtt(obj);	2776	i915_gem_object_put_pages_gtt(obj);
2777	drm_mm_put_block(obj->gtt_space);	2777	drm_mm_put_block(obj->gtt_space);
2778	obj->gtt_space = NULL;	2778	obj->gtt_space = NULL;
2779		2779
2780	if (i915_gem_evict_everything(dev, false))	2780	if (i915_gem_evict_everything(dev, false))
2781	return ret;	2781	return ret;
2782		2782
2783	goto search_free;	2783	goto search_free;
2784	}	2784	}
2785		2785
2786	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);	2786	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2787	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);	2787	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2788		2788
2789	/* Assert that the object is not currently in any GPU domain. As it	2789	/* Assert that the object is not currently in any GPU domain. As it
2790	* wasn't in the GTT, there shouldn't be any way it could have been in	2790	* wasn't in the GTT, there shouldn't be any way it could have been in
2791	* a GPU cache	2791	* a GPU cache
2792	*/	2792	*/
2793	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);	2793	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2794	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);	2794	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2795		2795
2796	obj->gtt_offset = obj->gtt_space->start;	2796	obj->gtt_offset = obj->gtt_space->start;
2797		2797
2798	fenceable =	2798	fenceable =
2799	obj->gtt_space->size == fence_size &&	2799	obj->gtt_space->size == fence_size &&
2800	(obj->gtt_space->start & (fence_alignment - 1)) == 0;	2800	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2801		2801
2802	mappable =	2802	mappable =
2803	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;	2803	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2804		2804
2805	obj->map_and_fenceable = mappable && fenceable;	2805	obj->map_and_fenceable = mappable && fenceable;
2806		2806
2807	trace_i915_gem_object_bind(obj, map_and_fenceable);	2807	trace_i915_gem_object_bind(obj, map_and_fenceable);
2808	return 0;	2808	return 0;
2809	}	2809	}
2810		2810
2811	void	2811	void
2812	i915_gem_clflush_object(struct drm_i915_gem_object *obj)	2812	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2813	{	2813	{
2814	/* If we don't have a page list set up, then we're not pinned	2814	/* If we don't have a page list set up, then we're not pinned
2815	* to GPU, and we can ignore the cache flush because it'll happen	2815	* to GPU, and we can ignore the cache flush because it'll happen
2816	* again at bind time.	2816	* again at bind time.
2817	*/	2817	*/
2818	if (obj->pages == NULL)	2818	if (obj->pages == NULL)
2819	return;	2819	return;
2820		2820
2821	/* If the GPU is snooping the contents of the CPU cache,	2821	/* If the GPU is snooping the contents of the CPU cache,
2822	* we do not need to manually clear the CPU cache lines. However,	2822	* we do not need to manually clear the CPU cache lines. However,
2823	* the caches are only snooped when the render cache is	2823	* the caches are only snooped when the render cache is
2824	* flushed/invalidated. As we always have to emit invalidations	2824	* flushed/invalidated. As we always have to emit invalidations
2825	* and flushes when moving into and out of the RENDER domain, correct	2825	* and flushes when moving into and out of the RENDER domain, correct
2826	* snooping behaviour occurs naturally as the result of our domain	2826	* snooping behaviour occurs naturally as the result of our domain
2827	* tracking.	2827	* tracking.
2828	*/	2828	*/
2829	if (obj->cache_level != I915_CACHE_NONE)	2829	if (obj->cache_level != I915_CACHE_NONE)
2830	return;	2830	return;
2831		2831
2832	trace_i915_gem_object_clflush(obj);	2832	trace_i915_gem_object_clflush(obj);
2833		2833
2834	drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);	2834	drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2835	}	2835	}
2836		2836
2837	/** Flushes any GPU write domain for the object if it's dirty. */	2837	/** Flushes any GPU write domain for the object if it's dirty. */
2838	static int	2838	static int
2839	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)	2839	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2840	{	2840	{
2841	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)	2841	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2842	return 0;	2842	return 0;
2843		2843
2844	/* Queue the GPU write cache flushing we need. */	2844	/* Queue the GPU write cache flushing we need. */
2845	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);	2845	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2846	}	2846	}
2847		2847
2848	/** Flushes the GTT write domain for the object if it's dirty. */	2848	/** Flushes the GTT write domain for the object if it's dirty. */
2849	static void	2849	static void
2850	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)	2850	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2851	{	2851	{
2852	uint32_t old_write_domain;	2852	uint32_t old_write_domain;
2853		2853
2854	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)	2854	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2855	return;	2855	return;
2856		2856
2857	/* No actual flushing is required for the GTT write domain. Writes	2857	/* No actual flushing is required for the GTT write domain. Writes
2858	* to it immediately go to main memory as far as we know, so there's	2858	* to it immediately go to main memory as far as we know, so there's
2859	* no chipset flush. It also doesn't land in render cache.	2859	* no chipset flush. It also doesn't land in render cache.
2860	*	2860	*
2861	* However, we do have to enforce the order so that all writes through	2861	* However, we do have to enforce the order so that all writes through
2862	* the GTT land before any writes to the device, such as updates to	2862	* the GTT land before any writes to the device, such as updates to
2863	* the GATT itself.	2863	* the GATT itself.
2864	*/	2864	*/
2865	wmb();	2865	wmb();
2866		2866
2867	old_write_domain = obj->base.write_domain;	2867	old_write_domain = obj->base.write_domain;
2868	obj->base.write_domain = 0;	2868	obj->base.write_domain = 0;
2869		2869
2870	trace_i915_gem_object_change_domain(obj,	2870	trace_i915_gem_object_change_domain(obj,
2871	obj->base.read_domains,	2871	obj->base.read_domains,
2872	old_write_domain);	2872	old_write_domain);
2873	}	2873	}
2874		2874
2875	/** Flushes the CPU write domain for the object if it's dirty. */	2875	/** Flushes the CPU write domain for the object if it's dirty. */
2876	static void	2876	static void
2877	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)	2877	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2878	{	2878	{
2879	uint32_t old_write_domain;	2879	uint32_t old_write_domain;
2880		2880
2881	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)	2881	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2882	return;	2882	return;
2883		2883
2884	i915_gem_clflush_object(obj);	2884	i915_gem_clflush_object(obj);
2885	intel_gtt_chipset_flush();	2885	intel_gtt_chipset_flush();
2886	old_write_domain = obj->base.write_domain;	2886	old_write_domain = obj->base.write_domain;
2887	obj->base.write_domain = 0;	2887	obj->base.write_domain = 0;
2888		2888
2889	trace_i915_gem_object_change_domain(obj,	2889	trace_i915_gem_object_change_domain(obj,
2890	obj->base.read_domains,	2890	obj->base.read_domains,
2891	old_write_domain);	2891	old_write_domain);
2892	}	2892	}
2893		2893
2894	/**	2894	/**
2895	* Moves a single object to the GTT read, and possibly write domain.	2895	* Moves a single object to the GTT read, and possibly write domain.
2896	*	2896	*
2897	* This function returns when the move is complete, including waiting on	2897	* This function returns when the move is complete, including waiting on
2898	* flushes to occur.	2898	* flushes to occur.
2899	*/	2899	*/
2900	int	2900	int
2901	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)	2901	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2902	{	2902	{
2903	uint32_t old_write_domain, old_read_domains;	2903	uint32_t old_write_domain, old_read_domains;
2904	int ret;	2904	int ret;
2905		2905
2906	/* Not valid to be called on unbound objects. */	2906	/* Not valid to be called on unbound objects. */
2907	if (obj->gtt_space == NULL)	2907	if (obj->gtt_space == NULL)
2908	return -EINVAL;	2908	return -EINVAL;
2909		2909
2910	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)	2910	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2911	return 0;	2911	return 0;
2912		2912
2913	ret = i915_gem_object_flush_gpu_write_domain(obj);	2913	ret = i915_gem_object_flush_gpu_write_domain(obj);
2914	if (ret)	2914	if (ret)
2915	return ret;	2915	return ret;
2916		2916
2917	if (obj->pending_gpu_write \|\| write) {	2917	if (obj->pending_gpu_write \|\| write) {
2918	ret = i915_gem_object_wait_rendering(obj);	2918	ret = i915_gem_object_wait_rendering(obj);
2919	if (ret)	2919	if (ret)
2920	return ret;	2920	return ret;
2921	}	2921	}
2922		2922
2923	i915_gem_object_flush_cpu_write_domain(obj);	2923	i915_gem_object_flush_cpu_write_domain(obj);
2924		2924
2925	old_write_domain = obj->base.write_domain;	2925	old_write_domain = obj->base.write_domain;
2926	old_read_domains = obj->base.read_domains;	2926	old_read_domains = obj->base.read_domains;
2927		2927
2928	/* It should now be out of any other write domains, and we can update	2928	/* It should now be out of any other write domains, and we can update
2929	* the domain values for our changes.	2929	* the domain values for our changes.
2930	*/	2930	*/
2931	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);	2931	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2932	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;	2932	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2933	if (write) {	2933	if (write) {
2934	obj->base.read_domains = I915_GEM_DOMAIN_GTT;	2934	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2935	obj->base.write_domain = I915_GEM_DOMAIN_GTT;	2935	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2936	obj->dirty = 1;	2936	obj->dirty = 1;
2937	}	2937	}
2938		2938
2939	trace_i915_gem_object_change_domain(obj,	2939	trace_i915_gem_object_change_domain(obj,
2940	old_read_domains,	2940	old_read_domains,
2941	old_write_domain);	2941	old_write_domain);
2942		2942
2943	return 0;	2943	return 0;
2944	}	2944	}
2945		2945
2946	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,	2946	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2947	enum i915_cache_level cache_level)	2947	enum i915_cache_level cache_level)
2948	{	2948	{
2949	int ret;	2949	int ret;
2950		2950
2951	if (obj->cache_level == cache_level)	2951	if (obj->cache_level == cache_level)
2952	return 0;	2952	return 0;
2953		2953
2954	if (obj->pin_count) {	2954	if (obj->pin_count) {
2955	DRM_DEBUG("can not change the cache level of pinned objects\n");	2955	DRM_DEBUG("can not change the cache level of pinned objects\n");
2956	return -EBUSY;	2956	return -EBUSY;
2957	}	2957	}
2958		2958
2959	if (obj->gtt_space) {	2959	if (obj->gtt_space) {
2960	ret = i915_gem_object_finish_gpu(obj);	2960	ret = i915_gem_object_finish_gpu(obj);
2961	if (ret)	2961	if (ret)
2962	return ret;	2962	return ret;
2963		2963
2964	i915_gem_object_finish_gtt(obj);	2964	i915_gem_object_finish_gtt(obj);
2965		2965
2966	/* Before SandyBridge, you could not use tiling or fence	2966	/* Before SandyBridge, you could not use tiling or fence
2967	* registers with snooped memory, so relinquish any fences	2967	* registers with snooped memory, so relinquish any fences
2968	* currently pointing to our region in the aperture.	2968	* currently pointing to our region in the aperture.
2969	*/	2969	*/
2970	if (INTEL_INFO(obj->base.dev)->gen < 6) {	2970	if (INTEL_INFO(obj->base.dev)->gen < 6) {
2971	ret = i915_gem_object_put_fence(obj);	2971	ret = i915_gem_object_put_fence(obj);
2972	if (ret)	2972	if (ret)
2973	return ret;	2973	return ret;
2974	}	2974	}
2975		2975
2976	i915_gem_gtt_rebind_object(obj, cache_level);	2976	i915_gem_gtt_rebind_object(obj, cache_level);
2977	}	2977	}
2978		2978
2979	if (cache_level == I915_CACHE_NONE) {	2979	if (cache_level == I915_CACHE_NONE) {
2980	u32 old_read_domains, old_write_domain;	2980	u32 old_read_domains, old_write_domain;
2981		2981
2982	/* If we're coming from LLC cached, then we haven't	2982	/* If we're coming from LLC cached, then we haven't
2983	* actually been tracking whether the data is in the	2983	* actually been tracking whether the data is in the
2984	* CPU cache or not, since we only allow one bit set	2984	* CPU cache or not, since we only allow one bit set
2985	* in obj->write_domain and have been skipping the clflushes.	2985	* in obj->write_domain and have been skipping the clflushes.
2986	* Just set it to the CPU cache for now.	2986	* Just set it to the CPU cache for now.
2987	*/	2987	*/
2988	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);	2988	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2989	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);	2989	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2990		2990
2991	old_read_domains = obj->base.read_domains;	2991	old_read_domains = obj->base.read_domains;
2992	old_write_domain = obj->base.write_domain;	2992	old_write_domain = obj->base.write_domain;
2993		2993
2994	obj->base.read_domains = I915_GEM_DOMAIN_CPU;	2994	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2995	obj->base.write_domain = I915_GEM_DOMAIN_CPU;	2995	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2996		2996
2997	trace_i915_gem_object_change_domain(obj,	2997	trace_i915_gem_object_change_domain(obj,
2998	old_read_domains,	2998	old_read_domains,
2999	old_write_domain);	2999	old_write_domain);
3000	}	3000	}
3001		3001
3002	obj->cache_level = cache_level;	3002	obj->cache_level = cache_level;
3003	return 0;	3003	return 0;
3004	}	3004	}
3005		3005
3006	/*	3006	/*
3007	* Prepare buffer for display plane (scanout, cursors, etc).	3007	* Prepare buffer for display plane (scanout, cursors, etc).
3008	* Can be called from an uninterruptible phase (modesetting) and allows	3008	* Can be called from an uninterruptible phase (modesetting) and allows
3009	* any flushes to be pipelined (for pageflips).	3009	* any flushes to be pipelined (for pageflips).
3010	*	3010	*
3011	* For the display plane, we want to be in the GTT but out of any write	3011	* For the display plane, we want to be in the GTT but out of any write
3012	* domains. So in many ways this looks like set_to_gtt_domain() apart from the	3012	* domains. So in many ways this looks like set_to_gtt_domain() apart from the
3013	* ability to pipeline the waits, pinning and any additional subtleties	3013	* ability to pipeline the waits, pinning and any additional subtleties
3014	* that may differentiate the display plane from ordinary buffers.	3014	* that may differentiate the display plane from ordinary buffers.
3015	*/	3015	*/
3016	int	3016	int
3017	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,	3017	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3018	u32 alignment,	3018	u32 alignment,
3019	struct intel_ring_buffer *pipelined)	3019	struct intel_ring_buffer *pipelined)
3020	{	3020	{
3021	u32 old_read_domains, old_write_domain;	3021	u32 old_read_domains, old_write_domain;
3022	int ret;	3022	int ret;
3023		3023
3024	ret = i915_gem_object_flush_gpu_write_domain(obj);	3024	ret = i915_gem_object_flush_gpu_write_domain(obj);
3025	if (ret)	3025	if (ret)
3026	return ret;	3026	return ret;
3027		3027
3028	if (pipelined != obj->ring) {	3028	if (pipelined != obj->ring) {
3029	ret = i915_gem_object_wait_rendering(obj);	3029	ret = i915_gem_object_wait_rendering(obj);
3030	if (ret == -ERESTARTSYS)	3030	if (ret == -ERESTARTSYS)
3031	return ret;	3031	return ret;
3032	}	3032	}
3033		3033
3034	/* The display engine is not coherent with the LLC cache on gen6. As	3034	/* The display engine is not coherent with the LLC cache on gen6. As
3035	* a result, we make sure that the pinning that is about to occur is	3035	* a result, we make sure that the pinning that is about to occur is
3036	* done with uncached PTEs. This is lowest common denominator for all	3036	* done with uncached PTEs. This is lowest common denominator for all
3037	* chipsets.	3037	* chipsets.
3038	*	3038	*
3039	* However for gen6+, we could do better by using the GFDT bit instead	3039	* However for gen6+, we could do better by using the GFDT bit instead
3040	* of uncaching, which would allow us to flush all the LLC-cached data	3040	* of uncaching, which would allow us to flush all the LLC-cached data
3041	* with that bit in the PTE to main memory with just one PIPE_CONTROL.	3041	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
3042	*/	3042	*/
3043	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);	3043	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3044	if (ret)	3044	if (ret)
3045	return ret;	3045	return ret;
3046		3046
3047	/* As the user may map the buffer once pinned in the display plane	3047	/* As the user may map the buffer once pinned in the display plane
3048	* (e.g. libkms for the bootup splash), we have to ensure that we	3048	* (e.g. libkms for the bootup splash), we have to ensure that we
3049	* always use map_and_fenceable for all scanout buffers.	3049	* always use map_and_fenceable for all scanout buffers.
3050	*/	3050	*/
3051	ret = i915_gem_object_pin(obj, alignment, true);	3051	ret = i915_gem_object_pin(obj, alignment, true);
3052	if (ret)	3052	if (ret)
3053	return ret;	3053	return ret;
3054		3054
3055	i915_gem_object_flush_cpu_write_domain(obj);	3055	i915_gem_object_flush_cpu_write_domain(obj);
3056		3056
3057	old_write_domain = obj->base.write_domain;	3057	old_write_domain = obj->base.write_domain;
3058	old_read_domains = obj->base.read_domains;	3058	old_read_domains = obj->base.read_domains;
3059		3059
3060	/* It should now be out of any other write domains, and we can update	3060	/* It should now be out of any other write domains, and we can update
3061	* the domain values for our changes.	3061	* the domain values for our changes.
3062	*/	3062	*/
3063	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);	3063	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3064	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;	3064	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
3065		3065
3066	trace_i915_gem_object_change_domain(obj,	3066	trace_i915_gem_object_change_domain(obj,
3067	old_read_domains,	3067	old_read_domains,
3068	old_write_domain);	3068	old_write_domain);
3069		3069
3070	return 0;	3070	return 0;
3071	}	3071	}
3072		3072
3073	int	3073	int
3074	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)	3074	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3075	{	3075	{
3076	int ret;	3076	int ret;
3077		3077
3078	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)	3078	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3079	return 0;	3079	return 0;
3080		3080
3081	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {	3081	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3082	ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);	3082	ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3083	if (ret)	3083	if (ret)
3084	return ret;	3084	return ret;
3085	}	3085	}
3086		3086
3087	/* Ensure that we invalidate the GPU's caches and TLBs. */	3087	/* Ensure that we invalidate the GPU's caches and TLBs. */
3088	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;	3088	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3089		3089
3090	return i915_gem_object_wait_rendering(obj);	3090	return i915_gem_object_wait_rendering(obj);
3091	}	3091	}
3092		3092
3093	/**	3093	/**
3094	* Moves a single object to the CPU read, and possibly write domain.	3094	* Moves a single object to the CPU read, and possibly write domain.
3095	*	3095	*
3096	* This function returns when the move is complete, including waiting on	3096	* This function returns when the move is complete, including waiting on
3097	* flushes to occur.	3097	* flushes to occur.
3098	*/	3098	*/
3099	static int	3099	static int
3100	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)	3100	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3101	{	3101	{
3102	uint32_t old_write_domain, old_read_domains;	3102	uint32_t old_write_domain, old_read_domains;
3103	int ret;	3103	int ret;
3104		3104
3105	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)	3105	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3106	return 0;	3106	return 0;
3107		3107
3108	ret = i915_gem_object_flush_gpu_write_domain(obj);	3108	ret = i915_gem_object_flush_gpu_write_domain(obj);
3109	if (ret)	3109	if (ret)
3110	return ret;	3110	return ret;
3111		3111
3112	ret = i915_gem_object_wait_rendering(obj);	3112	ret = i915_gem_object_wait_rendering(obj);
3113	if (ret)	3113	if (ret)
3114	return ret;	3114	return ret;
3115		3115
3116	i915_gem_object_flush_gtt_write_domain(obj);	3116	i915_gem_object_flush_gtt_write_domain(obj);
3117		3117
3118	/* If we have a partially-valid cache of the object in the CPU,	3118	/* If we have a partially-valid cache of the object in the CPU,
3119	* finish invalidating it and free the per-page flags.	3119	* finish invalidating it and free the per-page flags.
3120	*/	3120	*/
3121	i915_gem_object_set_to_full_cpu_read_domain(obj);	3121	i915_gem_object_set_to_full_cpu_read_domain(obj);
3122		3122
3123	old_write_domain = obj->base.write_domain;	3123	old_write_domain = obj->base.write_domain;
3124	old_read_domains = obj->base.read_domains;	3124	old_read_domains = obj->base.read_domains;
3125		3125
3126	/* Flush the CPU cache if it's still invalid. */	3126	/* Flush the CPU cache if it's still invalid. */
3127	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {	3127	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3128	i915_gem_clflush_object(obj);	3128	i915_gem_clflush_object(obj);
3129		3129
3130	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;	3130	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
3131	}	3131	}
3132		3132
3133	/* It should now be out of any other write domains, and we can update	3133	/* It should now be out of any other write domains, and we can update
3134	* the domain values for our changes.	3134	* the domain values for our changes.
3135	*/	3135	*/
3136	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);	3136	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3137		3137
3138	/* If we're writing through the CPU, then the GPU read domains will	3138	/* If we're writing through the CPU, then the GPU read domains will
3139	* need to be invalidated at next use.	3139	* need to be invalidated at next use.
3140	*/	3140	*/
3141	if (write) {	3141	if (write) {
3142	obj->base.read_domains = I915_GEM_DOMAIN_CPU;	3142	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3143	obj->base.write_domain = I915_GEM_DOMAIN_CPU;	3143	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3144	}	3144	}
3145		3145
3146	trace_i915_gem_object_change_domain(obj,	3146	trace_i915_gem_object_change_domain(obj,
3147	old_read_domains,	3147	old_read_domains,
3148	old_write_domain);	3148	old_write_domain);
3149		3149
3150	return 0;	3150	return 0;
3151	}	3151	}
3152		3152
3153	/**	3153	/**
3154	* Moves the object from a partially CPU read to a full one.	3154	* Moves the object from a partially CPU read to a full one.
3155	*	3155	*
3156	* Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),	3156	* Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3157	* and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).	3157	* and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3158	*/	3158	*/
3159	static void	3159	static void
3160	i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)	3160	i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3161	{	3161	{
3162	if (!obj->page_cpu_valid)	3162	if (!obj->page_cpu_valid)
3163	return;	3163	return;
3164		3164
3165	/* If we're partially in the CPU read domain, finish moving it in.	3165	/* If we're partially in the CPU read domain, finish moving it in.
3166	*/	3166	*/
3167	if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {	3167	if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3168	int i;	3168	int i;
3169		3169
3170	for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {	3170	for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3171	if (obj->page_cpu_valid[i])	3171	if (obj->page_cpu_valid[i])
3172	continue;	3172	continue;
3173	drm_clflush_pages(obj->pages + i, 1);	3173	drm_clflush_pages(obj->pages + i, 1);
3174	}	3174	}
3175	}	3175	}
3176		3176
3177	/* Free the page_cpu_valid mappings which are now stale, whether	3177	/* Free the page_cpu_valid mappings which are now stale, whether
3178	* or not we've got I915_GEM_DOMAIN_CPU.	3178	* or not we've got I915_GEM_DOMAIN_CPU.
3179	*/	3179	*/
3180	kfree(obj->page_cpu_valid);	3180	kfree(obj->page_cpu_valid);
3181	obj->page_cpu_valid = NULL;	3181	obj->page_cpu_valid = NULL;
3182	}	3182	}
3183		3183
3184	/**	3184	/**
3185	* Set the CPU read domain on a range of the object.	3185	* Set the CPU read domain on a range of the object.
3186	*	3186	*
3187	* The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's	3187	* The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3188	* not entirely valid. The page_cpu_valid member of the object flags which	3188	* not entirely valid. The page_cpu_valid member of the object flags which
3189	* pages have been flushed, and will be respected by	3189	* pages have been flushed, and will be respected by
3190	* i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping	3190	* i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3191	* of the whole object.	3191	* of the whole object.
3192	*	3192	*
3193	* This function returns when the move is complete, including waiting on	3193	* This function returns when the move is complete, including waiting on
3194	* flushes to occur.	3194	* flushes to occur.
3195	*/	3195	*/
3196	static int	3196	static int
3197	i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,	3197	i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3198	uint64_t offset, uint64_t size)	3198	uint64_t offset, uint64_t size)
3199	{	3199	{
3200	uint32_t old_read_domains;	3200	uint32_t old_read_domains;
3201	int i, ret;	3201	int i, ret;
3202		3202
3203	if (offset == 0 && size == obj->base.size)	3203	if (offset == 0 && size == obj->base.size)
3204	return i915_gem_object_set_to_cpu_domain(obj, 0);	3204	return i915_gem_object_set_to_cpu_domain(obj, 0);
3205		3205
3206	ret = i915_gem_object_flush_gpu_write_domain(obj);	3206	ret = i915_gem_object_flush_gpu_write_domain(obj);
3207	if (ret)	3207	if (ret)
3208	return ret;	3208	return ret;
3209		3209
3210	ret = i915_gem_object_wait_rendering(obj);	3210	ret = i915_gem_object_wait_rendering(obj);
3211	if (ret)	3211	if (ret)
3212	return ret;	3212	return ret;
3213		3213
3214	i915_gem_object_flush_gtt_write_domain(obj);	3214	i915_gem_object_flush_gtt_write_domain(obj);
3215		3215
3216	/* If we're already fully in the CPU read domain, we're done. */	3216	/* If we're already fully in the CPU read domain, we're done. */
3217	if (obj->page_cpu_valid == NULL &&	3217	if (obj->page_cpu_valid == NULL &&
3218	(obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)	3218	(obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3219	return 0;	3219	return 0;
3220		3220
3221	/* Otherwise, create/clear the per-page CPU read domain flag if we're	3221	/* Otherwise, create/clear the per-page CPU read domain flag if we're
3222	* newly adding I915_GEM_DOMAIN_CPU	3222	* newly adding I915_GEM_DOMAIN_CPU
3223	*/	3223	*/
3224	if (obj->page_cpu_valid == NULL) {	3224	if (obj->page_cpu_valid == NULL) {
3225	obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,	3225	obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3226	GFP_KERNEL);	3226	GFP_KERNEL);
3227	if (obj->page_cpu_valid == NULL)	3227	if (obj->page_cpu_valid == NULL)
3228	return -ENOMEM;	3228	return -ENOMEM;
3229	} else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)	3229	} else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3230	memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);	3230	memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3231		3231
3232	/* Flush the cache on any pages that are still invalid from the CPU's	3232	/* Flush the cache on any pages that are still invalid from the CPU's
3233	* perspective.	3233	* perspective.
3234	*/	3234	*/
3235	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;	3235	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3236	i++) {	3236	i++) {
3237	if (obj->page_cpu_valid[i])	3237	if (obj->page_cpu_valid[i])
3238	continue;	3238	continue;
3239		3239
3240	drm_clflush_pages(obj->pages + i, 1);	3240	drm_clflush_pages(obj->pages + i, 1);
3241		3241
3242	obj->page_cpu_valid[i] = 1;	3242	obj->page_cpu_valid[i] = 1;
3243	}	3243	}
3244		3244
3245	/* It should now be out of any other write domains, and we can update	3245	/* It should now be out of any other write domains, and we can update
3246	* the domain values for our changes.	3246	* the domain values for our changes.
3247	*/	3247	*/
3248	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);	3248	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3249		3249
3250	old_read_domains = obj->base.read_domains;	3250	old_read_domains = obj->base.read_domains;
3251	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;	3251	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
3252		3252
3253	trace_i915_gem_object_change_domain(obj,	3253	trace_i915_gem_object_change_domain(obj,
3254	old_read_domains,	3254	old_read_domains,
3255	obj->base.write_domain);	3255	obj->base.write_domain);
3256		3256
3257	return 0;	3257	return 0;
3258	}	3258	}
3259		3259
3260	/* Throttle our rendering by waiting until the ring has completed our requests	3260	/* Throttle our rendering by waiting until the ring has completed our requests
3261	* emitted over 20 msec ago.	3261	* emitted over 20 msec ago.
3262	*	3262	*
3263	* Note that if we were to use the current jiffies each time around the loop,	3263	* Note that if we were to use the current jiffies each time around the loop,
3264	* we wouldn't escape the function with any frames outstanding if the time to	3264	* we wouldn't escape the function with any frames outstanding if the time to
3265	* render a frame was over 20ms.	3265	* render a frame was over 20ms.
3266	*	3266	*
3267	* This should get us reasonable parallelism between CPU and GPU but also	3267	* This should get us reasonable parallelism between CPU and GPU but also
3268	* relatively low latency when blocking on a particular request to finish.	3268	* relatively low latency when blocking on a particular request to finish.
3269	*/	3269	*/
3270	static int	3270	static int
3271	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)	3271	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
3272	{	3272	{
3273	struct drm_i915_private *dev_priv = dev->dev_private;	3273	struct drm_i915_private *dev_priv = dev->dev_private;
3274	struct drm_i915_file_private *file_priv = file->driver_priv;	3274	struct drm_i915_file_private *file_priv = file->driver_priv;
3275	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);	3275	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3276	struct drm_i915_gem_request *request;	3276	struct drm_i915_gem_request *request;
3277	struct intel_ring_buffer *ring = NULL;	3277	struct intel_ring_buffer *ring = NULL;
3278	u32 seqno = 0;	3278	u32 seqno = 0;
3279	int ret;	3279	int ret;
3280		3280
3281	if (atomic_read(&dev_priv->mm.wedged))	3281	if (atomic_read(&dev_priv->mm.wedged))
3282	return -EIO;	3282	return -EIO;
3283		3283
3284	spin_lock(&file_priv->mm.lock);	3284	spin_lock(&file_priv->mm.lock);
3285	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {	3285	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3286	if (time_after_eq(request->emitted_jiffies, recent_enough))	3286	if (time_after_eq(request->emitted_jiffies, recent_enough))
3287	break;	3287	break;
3288		3288
3289	ring = request->ring;	3289	ring = request->ring;
3290	seqno = request->seqno;	3290	seqno = request->seqno;
3291	}	3291	}
3292	spin_unlock(&file_priv->mm.lock);	3292	spin_unlock(&file_priv->mm.lock);
3293		3293
3294	if (seqno == 0)	3294	if (seqno == 0)
3295	return 0;	3295	return 0;
3296		3296
3297	ret = 0;	3297	ret = 0;
3298	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {	3298	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3299	/* And wait for the seqno passing without holding any locks and	3299	/* And wait for the seqno passing without holding any locks and
3300	* causing extra latency for others. This is safe as the irq	3300	* causing extra latency for others. This is safe as the irq
3301	* generation is designed to be run atomically and so is	3301	* generation is designed to be run atomically and so is
3302	* lockless.	3302	* lockless.
3303	*/	3303	*/
3304	if (ring->irq_get(ring)) {	3304	if (ring->irq_get(ring)) {
3305	ret = wait_event_interruptible(ring->irq_queue,	3305	ret = wait_event_interruptible(ring->irq_queue,
3306	i915_seqno_passed(ring->get_seqno(ring), seqno)	3306	i915_seqno_passed(ring->get_seqno(ring), seqno)
3307	\|\| atomic_read(&dev_priv->mm.wedged));	3307	\|\| atomic_read(&dev_priv->mm.wedged));
3308	ring->irq_put(ring);	3308	ring->irq_put(ring);
3309		3309
3310	if (ret == 0 && atomic_read(&dev_priv->mm.wedged))	3310	if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3311	ret = -EIO;	3311	ret = -EIO;
3312	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),	3312	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
3313	seqno) \|\|	3313	seqno) \|\|
3314	atomic_read(&dev_priv->mm.wedged), 3000)) {	3314	atomic_read(&dev_priv->mm.wedged), 3000)) {
3315	ret = -EBUSY;	3315	ret = -EBUSY;
3316	}	3316	}
3317	}	3317	}
3318		3318
3319	if (ret == 0)	3319	if (ret == 0)
3320	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);	3320	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3321		3321
3322	return ret;	3322	return ret;
3323	}	3323	}
3324		3324
3325	int	3325	int
3326	i915_gem_object_pin(struct drm_i915_gem_object *obj,	3326	i915_gem_object_pin(struct drm_i915_gem_object *obj,
3327	uint32_t alignment,	3327	uint32_t alignment,
3328	bool map_and_fenceable)	3328	bool map_and_fenceable)
3329	{	3329	{
3330	struct drm_device *dev = obj->base.dev;	3330	struct drm_device *dev = obj->base.dev;
3331	struct drm_i915_private *dev_priv = dev->dev_private;	3331	struct drm_i915_private *dev_priv = dev->dev_private;
3332	int ret;	3332	int ret;
3333		3333
3334	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);	3334	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3335	WARN_ON(i915_verify_lists(dev));	3335	WARN_ON(i915_verify_lists(dev));
3336		3336
3337	if (obj->gtt_space != NULL) {	3337	if (obj->gtt_space != NULL) {
3338	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|	3338	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
3339	(map_and_fenceable && !obj->map_and_fenceable)) {	3339	(map_and_fenceable && !obj->map_and_fenceable)) {
3340	WARN(obj->pin_count,	3340	WARN(obj->pin_count,
3341	"bo is already pinned with incorrect alignment:"	3341	"bo is already pinned with incorrect alignment:"
3342	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"	3342	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3343	" obj->map_and_fenceable=%d\n",	3343	" obj->map_and_fenceable=%d\n",
3344	obj->gtt_offset, alignment,	3344	obj->gtt_offset, alignment,
3345	map_and_fenceable,	3345	map_and_fenceable,
3346	obj->map_and_fenceable);	3346	obj->map_and_fenceable);
3347	ret = i915_gem_object_unbind(obj);	3347	ret = i915_gem_object_unbind(obj);
3348	if (ret)	3348	if (ret)
3349	return ret;	3349	return ret;
3350	}	3350	}
3351	}	3351	}
3352		3352
3353	if (obj->gtt_space == NULL) {	3353	if (obj->gtt_space == NULL) {
3354	ret = i915_gem_object_bind_to_gtt(obj, alignment,	3354	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3355	map_and_fenceable);	3355	map_and_fenceable);
3356	if (ret)	3356	if (ret)
3357	return ret;	3357	return ret;
3358	}	3358	}
3359		3359
3360	if (obj->pin_count++ == 0) {	3360	if (obj->pin_count++ == 0) {
3361	if (!obj->active)	3361	if (!obj->active)
3362	list_move_tail(&obj->mm_list,	3362	list_move_tail(&obj->mm_list,
3363	&dev_priv->mm.pinned_list);	3363	&dev_priv->mm.pinned_list);
3364	}	3364	}
3365	obj->pin_mappable \|= map_and_fenceable;	3365	obj->pin_mappable \|= map_and_fenceable;
3366		3366
3367	WARN_ON(i915_verify_lists(dev));	3367	WARN_ON(i915_verify_lists(dev));
3368	return 0;	3368	return 0;
3369	}	3369	}
3370		3370
3371	void	3371	void
3372	i915_gem_object_unpin(struct drm_i915_gem_object *obj)	3372	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3373	{	3373	{
3374	struct drm_device *dev = obj->base.dev;	3374	struct drm_device *dev = obj->base.dev;
3375	drm_i915_private_t *dev_priv = dev->dev_private;	3375	drm_i915_private_t *dev_priv = dev->dev_private;
3376		3376
3377	WARN_ON(i915_verify_lists(dev));	3377	WARN_ON(i915_verify_lists(dev));
3378	BUG_ON(obj->pin_count == 0);	3378	BUG_ON(obj->pin_count == 0);
3379	BUG_ON(obj->gtt_space == NULL);	3379	BUG_ON(obj->gtt_space == NULL);
3380		3380
3381	if (--obj->pin_count == 0) {	3381	if (--obj->pin_count == 0) {
3382	if (!obj->active)	3382	if (!obj->active)
3383	list_move_tail(&obj->mm_list,	3383	list_move_tail(&obj->mm_list,
3384	&dev_priv->mm.inactive_list);	3384	&dev_priv->mm.inactive_list);
3385	obj->pin_mappable = false;	3385	obj->pin_mappable = false;
3386	}	3386	}
3387	WARN_ON(i915_verify_lists(dev));	3387	WARN_ON(i915_verify_lists(dev));
3388	}	3388	}
3389		3389
3390	int	3390	int
3391	i915_gem_pin_ioctl(struct drm_device dev, void data,	3391	i915_gem_pin_ioctl(struct drm_device dev, void data,
3392	struct drm_file *file)	3392	struct drm_file *file)
3393	{	3393	{
3394	struct drm_i915_gem_pin *args = data;	3394	struct drm_i915_gem_pin *args = data;
3395	struct drm_i915_gem_object *obj;	3395	struct drm_i915_gem_object *obj;
3396	int ret;	3396	int ret;
3397		3397
3398	ret = i915_mutex_lock_interruptible(dev);	3398	ret = i915_mutex_lock_interruptible(dev);
3399	if (ret)	3399	if (ret)
3400	return ret;	3400	return ret;
3401		3401
3402	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	3402	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3403	if (&obj->base == NULL) {	3403	if (&obj->base == NULL) {
3404	ret = -ENOENT;	3404	ret = -ENOENT;
3405	goto unlock;	3405	goto unlock;
3406	}	3406	}
3407		3407
3408	if (obj->madv != I915_MADV_WILLNEED) {	3408	if (obj->madv != I915_MADV_WILLNEED) {
3409	DRM_ERROR("Attempting to pin a purgeable buffer\n");	3409	DRM_ERROR("Attempting to pin a purgeable buffer\n");
3410	ret = -EINVAL;	3410	ret = -EINVAL;
3411	goto out;	3411	goto out;
3412	}	3412	}
3413		3413
3414	if (obj->pin_filp != NULL && obj->pin_filp != file) {	3414	if (obj->pin_filp != NULL && obj->pin_filp != file) {
3415	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",	3415	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3416	args->handle);	3416	args->handle);
3417	ret = -EINVAL;	3417	ret = -EINVAL;
3418	goto out;	3418	goto out;
3419	}	3419	}
3420		3420
3421	obj->user_pin_count++;	3421	obj->user_pin_count++;
3422	obj->pin_filp = file;	3422	obj->pin_filp = file;
3423	if (obj->user_pin_count == 1) {	3423	if (obj->user_pin_count == 1) {
3424	ret = i915_gem_object_pin(obj, args->alignment, true);	3424	ret = i915_gem_object_pin(obj, args->alignment, true);
3425	if (ret)	3425	if (ret)
3426	goto out;	3426	goto out;
3427	}	3427	}
3428		3428
3429	/* XXX - flush the CPU caches for pinned objects	3429	/* XXX - flush the CPU caches for pinned objects
3430	* as the X server doesn't manage domains yet	3430	* as the X server doesn't manage domains yet
3431	*/	3431	*/
3432	i915_gem_object_flush_cpu_write_domain(obj);	3432	i915_gem_object_flush_cpu_write_domain(obj);
3433	args->offset = obj->gtt_offset;	3433	args->offset = obj->gtt_offset;
3434	out:	3434	out:
3435	drm_gem_object_unreference(&obj->base);	3435	drm_gem_object_unreference(&obj->base);
3436	unlock:	3436	unlock:
3437	mutex_unlock(&dev->struct_mutex);	3437	mutex_unlock(&dev->struct_mutex);
3438	return ret;	3438	return ret;
3439	}	3439	}
3440		3440
3441	int	3441	int
3442	i915_gem_unpin_ioctl(struct drm_device dev, void data,	3442	i915_gem_unpin_ioctl(struct drm_device dev, void data,
3443	struct drm_file *file)	3443	struct drm_file *file)
3444	{	3444	{
3445	struct drm_i915_gem_pin *args = data;	3445	struct drm_i915_gem_pin *args = data;
3446	struct drm_i915_gem_object *obj;	3446	struct drm_i915_gem_object *obj;
3447	int ret;	3447	int ret;
3448		3448
3449	ret = i915_mutex_lock_interruptible(dev);	3449	ret = i915_mutex_lock_interruptible(dev);
3450	if (ret)	3450	if (ret)
3451	return ret;	3451	return ret;
3452		3452
3453	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	3453	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3454	if (&obj->base == NULL) {	3454	if (&obj->base == NULL) {
3455	ret = -ENOENT;	3455	ret = -ENOENT;
3456	goto unlock;	3456	goto unlock;
3457	}	3457	}
3458		3458
3459	if (obj->pin_filp != file) {	3459	if (obj->pin_filp != file) {
3460	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",	3460	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3461	args->handle);	3461	args->handle);
3462	ret = -EINVAL;	3462	ret = -EINVAL;
3463	goto out;	3463	goto out;
3464	}	3464	}
3465	obj->user_pin_count--;	3465	obj->user_pin_count--;
3466	if (obj->user_pin_count == 0) {	3466	if (obj->user_pin_count == 0) {
3467	obj->pin_filp = NULL;	3467	obj->pin_filp = NULL;
3468	i915_gem_object_unpin(obj);	3468	i915_gem_object_unpin(obj);
3469	}	3469	}
3470		3470
3471	out:	3471	out:
3472	drm_gem_object_unreference(&obj->base);	3472	drm_gem_object_unreference(&obj->base);
3473	unlock:	3473	unlock:
3474	mutex_unlock(&dev->struct_mutex);	3474	mutex_unlock(&dev->struct_mutex);
3475	return ret;	3475	return ret;
3476	}	3476	}
3477		3477
3478	int	3478	int
3479	i915_gem_busy_ioctl(struct drm_device dev, void data,	3479	i915_gem_busy_ioctl(struct drm_device dev, void data,
3480	struct drm_file *file)	3480	struct drm_file *file)
3481	{	3481	{
3482	struct drm_i915_gem_busy *args = data;	3482	struct drm_i915_gem_busy *args = data;
3483	struct drm_i915_gem_object *obj;	3483	struct drm_i915_gem_object *obj;
3484	int ret;	3484	int ret;
3485		3485
3486	ret = i915_mutex_lock_interruptible(dev);	3486	ret = i915_mutex_lock_interruptible(dev);
3487	if (ret)	3487	if (ret)
3488	return ret;	3488	return ret;
3489		3489
3490	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));	3490	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3491	if (&obj->base == NULL) {	3491	if (&obj->base == NULL) {
3492	ret = -ENOENT;	3492	ret = -ENOENT;
3493	goto unlock;	3493	goto unlock;
3494	}	3494	}
3495		3495
3496	/* Count all active objects as busy, even if they are currently not used	3496	/* Count all active objects as busy, even if they are currently not used
3497	* by the gpu. Users of this interface expect objects to eventually	3497	* by the gpu. Users of this interface expect objects to eventually
3498	* become non-busy without any further actions, therefore emit any	3498	* become non-busy without any further actions, therefore emit any
3499	* necessary flushes here.	3499	* necessary flushes here.
3500	*/	3500	*/
3501	args->busy = obj->active;	3501	args->busy = obj->active;
3502	if (args->busy) {	3502	if (args->busy) {
3503	/* Unconditionally flush objects, even when the gpu still uses this	3503	/* Unconditionally flush objects, even when the gpu still uses this
3504	* object. Userspace calling this function indicates that it wants to	3504	* object. Userspace calling this function indicates that it wants to
3505	* use this buffer rather sooner than later, so issuing the required	3505	* use this buffer rather sooner than later, so issuing the required
3506	* flush earlier is beneficial.	3506	* flush earlier is beneficial.
3507	*/	3507	*/
3508	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {	3508	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3509	ret = i915_gem_flush_ring(obj->ring,	3509	ret = i915_gem_flush_ring(obj->ring,
3510	0, obj->base.write_domain);	3510	0, obj->base.write_domain);
3511	} else if (obj->ring->outstanding_lazy_request ==	3511	} else if (obj->ring->outstanding_lazy_request ==
3512	obj->last_rendering_seqno) {	3512	obj->last_rendering_seqno) {
3513	struct drm_i915_gem_request *request;	3513	struct drm_i915_gem_request *request;
3514		3514
3515	/* This ring is not being cleared by active usage,	3515	/* This ring is not being cleared by active usage,
3516	* so emit a request to do so.	3516	* so emit a request to do so.
3517	*/	3517	*/
3518	request = kzalloc(sizeof(*request), GFP_KERNEL);	3518	request = kzalloc(sizeof(*request), GFP_KERNEL);
3519	if (request) {	3519	if (request) {
3520	ret = i915_add_request(obj->ring, NULL, request);	3520	ret = i915_add_request(obj->ring, NULL, request);
3521	if (ret)	3521	if (ret)
3522	kfree(request);	3522	kfree(request);
3523	} else	3523	} else
3524	ret = -ENOMEM;	3524	ret = -ENOMEM;
3525	}	3525	}
3526		3526
3527	/* Update the active list for the hardware's current position.	3527	/* Update the active list for the hardware's current position.
3528	* Otherwise this only updates on a delayed timer or when irqs	3528	* Otherwise this only updates on a delayed timer or when irqs
3529	* are actually unmasked, and our working set ends up being	3529	* are actually unmasked, and our working set ends up being
3530	* larger than required.	3530	* larger than required.
3531	*/	3531	*/
3532	i915_gem_retire_requests_ring(obj->ring);	3532	i915_gem_retire_requests_ring(obj->ring);
3533		3533
3534	args->busy = obj->active;	3534	args->busy = obj->active;
3535	}	3535	}
3536		3536
3537	drm_gem_object_unreference(&obj->base);	3537	drm_gem_object_unreference(&obj->base);
3538	unlock:	3538	unlock:
3539	mutex_unlock(&dev->struct_mutex);	3539	mutex_unlock(&dev->struct_mutex);
3540	return ret;	3540	return ret;
3541	}	3541	}
3542		3542
3543	int	3543	int
3544	i915_gem_throttle_ioctl(struct drm_device dev, void data,	3544	i915_gem_throttle_ioctl(struct drm_device dev, void data,
3545	struct drm_file *file_priv)	3545	struct drm_file *file_priv)
3546	{	3546	{
3547	return i915_gem_ring_throttle(dev, file_priv);	3547	return i915_gem_ring_throttle(dev, file_priv);
3548	}	3548	}
3549		3549
3550	int	3550	int
3551	i915_gem_madvise_ioctl(struct drm_device dev, void data,	3551	i915_gem_madvise_ioctl(struct drm_device dev, void data,
3552	struct drm_file *file_priv)	3552	struct drm_file *file_priv)
3553	{	3553	{
3554	struct drm_i915_gem_madvise *args = data;	3554	struct drm_i915_gem_madvise *args = data;
3555	struct drm_i915_gem_object *obj;	3555	struct drm_i915_gem_object *obj;
3556	int ret;	3556	int ret;
3557		3557
3558	switch (args->madv) {	3558	switch (args->madv) {
3559	case I915_MADV_DONTNEED:	3559	case I915_MADV_DONTNEED:
3560	case I915_MADV_WILLNEED:	3560	case I915_MADV_WILLNEED:
3561	break;	3561	break;
3562	default:	3562	default:
3563	return -EINVAL;	3563	return -EINVAL;
3564	}	3564	}
3565		3565
3566	ret = i915_mutex_lock_interruptible(dev);	3566	ret = i915_mutex_lock_interruptible(dev);
3567	if (ret)	3567	if (ret)
3568	return ret;	3568	return ret;
3569		3569
3570	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));	3570	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3571	if (&obj->base == NULL) {	3571	if (&obj->base == NULL) {
3572	ret = -ENOENT;	3572	ret = -ENOENT;
3573	goto unlock;	3573	goto unlock;
3574	}	3574	}
3575		3575
3576	if (obj->pin_count) {	3576	if (obj->pin_count) {
3577	ret = -EINVAL;	3577	ret = -EINVAL;
3578	goto out;	3578	goto out;
3579	}	3579	}
3580		3580
3581	if (obj->madv != __I915_MADV_PURGED)	3581	if (obj->madv != __I915_MADV_PURGED)
3582	obj->madv = args->madv;	3582	obj->madv = args->madv;
3583		3583
3584	/* if the object is no longer bound, discard its backing storage */	3584	/* if the object is no longer bound, discard its backing storage */
3585	if (i915_gem_object_is_purgeable(obj) &&	3585	if (i915_gem_object_is_purgeable(obj) &&
3586	obj->gtt_space == NULL)	3586	obj->gtt_space == NULL)
3587	i915_gem_object_truncate(obj);	3587	i915_gem_object_truncate(obj);
3588		3588
3589	args->retained = obj->madv != __I915_MADV_PURGED;	3589	args->retained = obj->madv != __I915_MADV_PURGED;
3590		3590
3591	out:	3591	out:
3592	drm_gem_object_unreference(&obj->base);	3592	drm_gem_object_unreference(&obj->base);
3593	unlock:	3593	unlock:
3594	mutex_unlock(&dev->struct_mutex);	3594	mutex_unlock(&dev->struct_mutex);
3595	return ret;	3595	return ret;
3596	}	3596	}
3597		3597
3598	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,	3598	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
3599	size_t size)	3599	size_t size)
3600	{	3600	{
3601	struct drm_i915_private *dev_priv = dev->dev_private;	3601	struct drm_i915_private *dev_priv = dev->dev_private;
3602	struct drm_i915_gem_object *obj;	3602	struct drm_i915_gem_object *obj;
3603	struct address_space *mapping;	3603	struct address_space *mapping;
3604		3604
3605	obj = kzalloc(sizeof(*obj), GFP_KERNEL);	3605	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3606	if (obj == NULL)	3606	if (obj == NULL)
3607	return NULL;	3607	return NULL;
3608		3608
3609	if (drm_gem_object_init(dev, &obj->base, size) != 0) {	3609	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3610	kfree(obj);	3610	kfree(obj);
3611	return NULL;	3611	return NULL;
3612	}	3612	}
3613		3613
3614	mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	3614	mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3615	mapping_set_gfp_mask(mapping, GFP_HIGHUSER \| __GFP_RECLAIMABLE);	3615	mapping_set_gfp_mask(mapping, GFP_HIGHUSER \| __GFP_RECLAIMABLE);
3616		3616
3617	i915_gem_info_add_obj(dev_priv, size);	3617	i915_gem_info_add_obj(dev_priv, size);
3618		3618
3619	obj->base.write_domain = I915_GEM_DOMAIN_CPU;	3619	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3620	obj->base.read_domains = I915_GEM_DOMAIN_CPU;	3620	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3621		3621
3622	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {	3622	if (HAS_LLC(dev)) {
3623	/* On Gen6, we can have the GPU use the LLC (the CPU	3623	/* On some devices, we can have the GPU use the LLC (the CPU
3624	* cache) for about a 10% performance improvement	3624	* cache) for about a 10% performance improvement
3625	* compared to uncached. Graphics requests other than	3625	* compared to uncached. Graphics requests other than
3626	* display scanout are coherent with the CPU in	3626	* display scanout are coherent with the CPU in
3627	* accessing this cache. This means in this mode we	3627	* accessing this cache. This means in this mode we
3628	* don't need to clflush on the CPU side, and on the	3628	* don't need to clflush on the CPU side, and on the
3629	* GPU side we only need to flush internal caches to	3629	* GPU side we only need to flush internal caches to
3630	* get data visible to the CPU.	3630	* get data visible to the CPU.
3631	*	3631	*
3632	* However, we maintain the display planes as UC, and so	3632	* However, we maintain the display planes as UC, and so
3633	* need to rebind when first used as such.	3633	* need to rebind when first used as such.
3634	*/	3634	*/
3635	obj->cache_level = I915_CACHE_LLC;	3635	obj->cache_level = I915_CACHE_LLC;
3636	} else	3636	} else
3637	obj->cache_level = I915_CACHE_NONE;	3637	obj->cache_level = I915_CACHE_NONE;
3638		3638
3639	obj->base.driver_private = NULL;	3639	obj->base.driver_private = NULL;
3640	obj->fence_reg = I915_FENCE_REG_NONE;	3640	obj->fence_reg = I915_FENCE_REG_NONE;
3641	INIT_LIST_HEAD(&obj->mm_list);	3641	INIT_LIST_HEAD(&obj->mm_list);
3642	INIT_LIST_HEAD(&obj->gtt_list);	3642	INIT_LIST_HEAD(&obj->gtt_list);
3643	INIT_LIST_HEAD(&obj->ring_list);	3643	INIT_LIST_HEAD(&obj->ring_list);
3644	INIT_LIST_HEAD(&obj->exec_list);	3644	INIT_LIST_HEAD(&obj->exec_list);
3645	INIT_LIST_HEAD(&obj->gpu_write_list);	3645	INIT_LIST_HEAD(&obj->gpu_write_list);
3646	obj->madv = I915_MADV_WILLNEED;	3646	obj->madv = I915_MADV_WILLNEED;
3647	/* Avoid an unnecessary call to unbind on the first bind. */	3647	/* Avoid an unnecessary call to unbind on the first bind. */
3648	obj->map_and_fenceable = true;	3648	obj->map_and_fenceable = true;
3649		3649
3650	return obj;	3650	return obj;
3651	}	3651	}
3652		3652
3653	int i915_gem_init_object(struct drm_gem_object *obj)	3653	int i915_gem_init_object(struct drm_gem_object *obj)
3654	{	3654	{
3655	BUG();	3655	BUG();
3656		3656
3657	return 0;	3657	return 0;
3658	}	3658	}
3659		3659
3660	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)	3660	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
3661	{	3661	{
3662	struct drm_device *dev = obj->base.dev;	3662	struct drm_device *dev = obj->base.dev;
3663	drm_i915_private_t *dev_priv = dev->dev_private;	3663	drm_i915_private_t *dev_priv = dev->dev_private;
3664	int ret;	3664	int ret;
3665		3665
3666	ret = i915_gem_object_unbind(obj);	3666	ret = i915_gem_object_unbind(obj);
3667	if (ret == -ERESTARTSYS) {	3667	if (ret == -ERESTARTSYS) {
3668	list_move(&obj->mm_list,	3668	list_move(&obj->mm_list,
3669	&dev_priv->mm.deferred_free_list);	3669	&dev_priv->mm.deferred_free_list);
3670	return;	3670	return;
3671	}	3671	}
3672		3672
3673	trace_i915_gem_object_destroy(obj);	3673	trace_i915_gem_object_destroy(obj);
3674		3674
3675	if (obj->base.map_list.map)	3675	if (obj->base.map_list.map)
3676	drm_gem_free_mmap_offset(&obj->base);	3676	drm_gem_free_mmap_offset(&obj->base);
3677		3677
3678	drm_gem_object_release(&obj->base);	3678	drm_gem_object_release(&obj->base);
3679	i915_gem_info_remove_obj(dev_priv, obj->base.size);	3679	i915_gem_info_remove_obj(dev_priv, obj->base.size);
3680		3680
3681	kfree(obj->page_cpu_valid);	3681	kfree(obj->page_cpu_valid);
3682	kfree(obj->bit_17);	3682	kfree(obj->bit_17);
3683	kfree(obj);	3683	kfree(obj);
3684	}	3684	}
3685		3685
3686	void i915_gem_free_object(struct drm_gem_object *gem_obj)	3686	void i915_gem_free_object(struct drm_gem_object *gem_obj)
3687	{	3687	{
3688	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);	3688	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3689	struct drm_device *dev = obj->base.dev;	3689	struct drm_device *dev = obj->base.dev;
3690		3690
3691	while (obj->pin_count > 0)	3691	while (obj->pin_count > 0)
3692	i915_gem_object_unpin(obj);	3692	i915_gem_object_unpin(obj);
3693		3693
3694	if (obj->phys_obj)	3694	if (obj->phys_obj)
3695	i915_gem_detach_phys_object(dev, obj);	3695	i915_gem_detach_phys_object(dev, obj);
3696		3696
3697	i915_gem_free_object_tail(obj);	3697	i915_gem_free_object_tail(obj);
3698	}	3698	}
3699		3699
3700	int	3700	int
3701	i915_gem_idle(struct drm_device *dev)	3701	i915_gem_idle(struct drm_device *dev)
3702	{	3702	{
3703	drm_i915_private_t *dev_priv = dev->dev_private;	3703	drm_i915_private_t *dev_priv = dev->dev_private;
3704	int ret;	3704	int ret;
3705		3705
3706	mutex_lock(&dev->struct_mutex);	3706	mutex_lock(&dev->struct_mutex);
3707		3707
3708	if (dev_priv->mm.suspended) {	3708	if (dev_priv->mm.suspended) {
3709	mutex_unlock(&dev->struct_mutex);	3709	mutex_unlock(&dev->struct_mutex);
3710	return 0;	3710	return 0;
3711	}	3711	}
3712		3712
3713	ret = i915_gpu_idle(dev);	3713	ret = i915_gpu_idle(dev);
3714	if (ret) {	3714	if (ret) {
3715	mutex_unlock(&dev->struct_mutex);	3715	mutex_unlock(&dev->struct_mutex);
3716	return ret;	3716	return ret;
3717	}	3717	}
3718		3718
3719	/* Under UMS, be paranoid and evict. */	3719	/* Under UMS, be paranoid and evict. */
3720	if (!drm_core_check_feature(dev, DRIVER_MODESET)) {	3720	if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
3721	ret = i915_gem_evict_inactive(dev, false);	3721	ret = i915_gem_evict_inactive(dev, false);
3722	if (ret) {	3722	if (ret) {
3723	mutex_unlock(&dev->struct_mutex);	3723	mutex_unlock(&dev->struct_mutex);
3724	return ret;	3724	return ret;
3725	}	3725	}
3726	}	3726	}
3727		3727
3728	i915_gem_reset_fences(dev);	3728	i915_gem_reset_fences(dev);
3729		3729
3730	/* Hack! Don't let anybody do execbuf while we don't control the chip.	3730	/* Hack! Don't let anybody do execbuf while we don't control the chip.
3731	* We need to replace this with a semaphore, or something.	3731	* We need to replace this with a semaphore, or something.
3732	* And not confound mm.suspended!	3732	* And not confound mm.suspended!
3733	*/	3733	*/
3734	dev_priv->mm.suspended = 1;	3734	dev_priv->mm.suspended = 1;
3735	del_timer_sync(&dev_priv->hangcheck_timer);	3735	del_timer_sync(&dev_priv->hangcheck_timer);
3736		3736
3737	i915_kernel_lost_context(dev);	3737	i915_kernel_lost_context(dev);
3738	i915_gem_cleanup_ringbuffer(dev);	3738	i915_gem_cleanup_ringbuffer(dev);
3739		3739
3740	mutex_unlock(&dev->struct_mutex);	3740	mutex_unlock(&dev->struct_mutex);
3741		3741
3742	/* Cancel the retire work handler, which should be idle now. */	3742	/* Cancel the retire work handler, which should be idle now. */
3743	cancel_delayed_work_sync(&dev_priv->mm.retire_work);	3743	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3744		3744
3745	return 0;	3745	return 0;
3746	}	3746	}
3747		3747
3748	int	3748	int
3749	i915_gem_init_ringbuffer(struct drm_device *dev)	3749	i915_gem_init_ringbuffer(struct drm_device *dev)
3750	{	3750	{
3751	drm_i915_private_t *dev_priv = dev->dev_private;	3751	drm_i915_private_t *dev_priv = dev->dev_private;
3752	int ret;	3752	int ret;
3753		3753
3754	ret = intel_init_render_ring_buffer(dev);	3754	ret = intel_init_render_ring_buffer(dev);
3755	if (ret)	3755	if (ret)
3756	return ret;	3756	return ret;
3757		3757
3758	if (HAS_BSD(dev)) {	3758	if (HAS_BSD(dev)) {
3759	ret = intel_init_bsd_ring_buffer(dev);	3759	ret = intel_init_bsd_ring_buffer(dev);
3760	if (ret)	3760	if (ret)
3761	goto cleanup_render_ring;	3761	goto cleanup_render_ring;
3762	}	3762	}
3763		3763
3764	if (HAS_BLT(dev)) {	3764	if (HAS_BLT(dev)) {
3765	ret = intel_init_blt_ring_buffer(dev);	3765	ret = intel_init_blt_ring_buffer(dev);
3766	if (ret)	3766	if (ret)
3767	goto cleanup_bsd_ring;	3767	goto cleanup_bsd_ring;
3768	}	3768	}
3769		3769
3770	dev_priv->next_seqno = 1;	3770	dev_priv->next_seqno = 1;
3771		3771
3772	return 0;	3772	return 0;
3773		3773
3774	cleanup_bsd_ring:	3774	cleanup_bsd_ring:
3775	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);	3775	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3776	cleanup_render_ring:	3776	cleanup_render_ring:
3777	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);	3777	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3778	return ret;	3778	return ret;
3779	}	3779	}
3780		3780
3781	void	3781	void
3782	i915_gem_cleanup_ringbuffer(struct drm_device *dev)	3782	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3783	{	3783	{
3784	drm_i915_private_t *dev_priv = dev->dev_private;	3784	drm_i915_private_t *dev_priv = dev->dev_private;
3785	int i;	3785	int i;
3786		3786
3787	for (i = 0; i < I915_NUM_RINGS; i++)	3787	for (i = 0; i < I915_NUM_RINGS; i++)
3788	intel_cleanup_ring_buffer(&dev_priv->ring[i]);	3788	intel_cleanup_ring_buffer(&dev_priv->ring[i]);
3789	}	3789	}
3790		3790
3791	int	3791	int
3792	i915_gem_entervt_ioctl(struct drm_device dev, void data,	3792	i915_gem_entervt_ioctl(struct drm_device dev, void data,
3793	struct drm_file *file_priv)	3793	struct drm_file *file_priv)
3794	{	3794	{
3795	drm_i915_private_t *dev_priv = dev->dev_private;	3795	drm_i915_private_t *dev_priv = dev->dev_private;
3796	int ret, i;	3796	int ret, i;
3797		3797
3798	if (drm_core_check_feature(dev, DRIVER_MODESET))	3798	if (drm_core_check_feature(dev, DRIVER_MODESET))
3799	return 0;	3799	return 0;
3800		3800
3801	if (atomic_read(&dev_priv->mm.wedged)) {	3801	if (atomic_read(&dev_priv->mm.wedged)) {
3802	DRM_ERROR("Reenabling wedged hardware, good luck\n");	3802	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3803	atomic_set(&dev_priv->mm.wedged, 0);	3803	atomic_set(&dev_priv->mm.wedged, 0);
3804	}	3804	}
3805		3805
3806	mutex_lock(&dev->struct_mutex);	3806	mutex_lock(&dev->struct_mutex);
3807	dev_priv->mm.suspended = 0;	3807	dev_priv->mm.suspended = 0;
3808		3808
3809	ret = i915_gem_init_ringbuffer(dev);	3809	ret = i915_gem_init_ringbuffer(dev);
3810	if (ret != 0) {	3810	if (ret != 0) {
3811	mutex_unlock(&dev->struct_mutex);	3811	mutex_unlock(&dev->struct_mutex);
3812	return ret;	3812	return ret;
3813	}	3813	}
3814		3814
3815	BUG_ON(!list_empty(&dev_priv->mm.active_list));	3815	BUG_ON(!list_empty(&dev_priv->mm.active_list));
3816	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));	3816	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3817	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));	3817	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3818	for (i = 0; i < I915_NUM_RINGS; i++) {	3818	for (i = 0; i < I915_NUM_RINGS; i++) {
3819	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));	3819	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3820	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));	3820	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3821	}	3821	}
3822	mutex_unlock(&dev->struct_mutex);	3822	mutex_unlock(&dev->struct_mutex);
3823		3823
3824	ret = drm_irq_install(dev);	3824	ret = drm_irq_install(dev);
3825	if (ret)	3825	if (ret)
3826	goto cleanup_ringbuffer;	3826	goto cleanup_ringbuffer;
3827		3827
3828	return 0;	3828	return 0;
3829		3829
3830	cleanup_ringbuffer:	3830	cleanup_ringbuffer:
3831	mutex_lock(&dev->struct_mutex);	3831	mutex_lock(&dev->struct_mutex);
3832	i915_gem_cleanup_ringbuffer(dev);	3832	i915_gem_cleanup_ringbuffer(dev);
3833	dev_priv->mm.suspended = 1;	3833	dev_priv->mm.suspended = 1;
3834	mutex_unlock(&dev->struct_mutex);	3834	mutex_unlock(&dev->struct_mutex);
3835		3835
3836	return ret;	3836	return ret;
3837	}	3837	}
3838		3838
3839	int	3839	int
3840	i915_gem_leavevt_ioctl(struct drm_device dev, void data,	3840	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
3841	struct drm_file *file_priv)	3841	struct drm_file *file_priv)
3842	{	3842	{
3843	if (drm_core_check_feature(dev, DRIVER_MODESET))	3843	if (drm_core_check_feature(dev, DRIVER_MODESET))
3844	return 0;	3844	return 0;
3845		3845
3846	drm_irq_uninstall(dev);	3846	drm_irq_uninstall(dev);
3847	return i915_gem_idle(dev);	3847	return i915_gem_idle(dev);
3848	}	3848	}
3849		3849
3850	void	3850	void
3851	i915_gem_lastclose(struct drm_device *dev)	3851	i915_gem_lastclose(struct drm_device *dev)
3852	{	3852	{
3853	int ret;	3853	int ret;
3854		3854
3855	if (drm_core_check_feature(dev, DRIVER_MODESET))	3855	if (drm_core_check_feature(dev, DRIVER_MODESET))
3856	return;	3856	return;
3857		3857
3858	ret = i915_gem_idle(dev);	3858	ret = i915_gem_idle(dev);
3859	if (ret)	3859	if (ret)
3860	DRM_ERROR("failed to idle hardware: %d\n", ret);	3860	DRM_ERROR("failed to idle hardware: %d\n", ret);
3861	}	3861	}
3862		3862
3863	static void	3863	static void
3864	init_ring_lists(struct intel_ring_buffer *ring)	3864	init_ring_lists(struct intel_ring_buffer *ring)
3865	{	3865	{
3866	INIT_LIST_HEAD(&ring->active_list);	3866	INIT_LIST_HEAD(&ring->active_list);
3867	INIT_LIST_HEAD(&ring->request_list);	3867	INIT_LIST_HEAD(&ring->request_list);
3868	INIT_LIST_HEAD(&ring->gpu_write_list);	3868	INIT_LIST_HEAD(&ring->gpu_write_list);
3869	}	3869	}
3870		3870
3871	void	3871	void
3872	i915_gem_load(struct drm_device *dev)	3872	i915_gem_load(struct drm_device *dev)
3873	{	3873	{
3874	int i;	3874	int i;
3875	drm_i915_private_t *dev_priv = dev->dev_private;	3875	drm_i915_private_t *dev_priv = dev->dev_private;
3876		3876
3877	INIT_LIST_HEAD(&dev_priv->mm.active_list);	3877	INIT_LIST_HEAD(&dev_priv->mm.active_list);
3878	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);	3878	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3879	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);	3879	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3880	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);	3880	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3881	INIT_LIST_HEAD(&dev_priv->mm.fence_list);	3881	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3882	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);	3882	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3883	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);	3883	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3884	for (i = 0; i < I915_NUM_RINGS; i++)	3884	for (i = 0; i < I915_NUM_RINGS; i++)
3885	init_ring_lists(&dev_priv->ring[i]);	3885	init_ring_lists(&dev_priv->ring[i]);
3886	for (i = 0; i < I915_MAX_NUM_FENCES; i++)	3886	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3887	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);	3887	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3888	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,	3888	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3889	i915_gem_retire_work_handler);	3889	i915_gem_retire_work_handler);
3890	init_completion(&dev_priv->error_completion);	3890	init_completion(&dev_priv->error_completion);
3891		3891
3892	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */	3892	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3893	if (IS_GEN3(dev)) {	3893	if (IS_GEN3(dev)) {
3894	u32 tmp = I915_READ(MI_ARB_STATE);	3894	u32 tmp = I915_READ(MI_ARB_STATE);
3895	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {	3895	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3896	/* arb state is a masked write, so set bit + bit in mask */	3896	/* arb state is a masked write, so set bit + bit in mask */
3897	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);	3897	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3898	I915_WRITE(MI_ARB_STATE, tmp);	3898	I915_WRITE(MI_ARB_STATE, tmp);
3899	}	3899	}
3900	}	3900	}
3901		3901
3902	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;	3902	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3903		3903
3904	/* Old X drivers will take 0-2 for front, back, depth buffers */	3904	/* Old X drivers will take 0-2 for front, back, depth buffers */
3905	if (!drm_core_check_feature(dev, DRIVER_MODESET))	3905	if (!drm_core_check_feature(dev, DRIVER_MODESET))
3906	dev_priv->fence_reg_start = 3;	3906	dev_priv->fence_reg_start = 3;
3907		3907
3908	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))	3908	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
3909	dev_priv->num_fence_regs = 16;	3909	dev_priv->num_fence_regs = 16;
3910	else	3910	else
3911	dev_priv->num_fence_regs = 8;	3911	dev_priv->num_fence_regs = 8;
3912		3912
3913	/* Initialize fence registers to zero */	3913	/* Initialize fence registers to zero */
3914	for (i = 0; i < dev_priv->num_fence_regs; i++) {	3914	for (i = 0; i < dev_priv->num_fence_regs; i++) {
3915	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);	3915	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
3916	}	3916	}
3917		3917
3918	i915_gem_detect_bit_6_swizzle(dev);	3918	i915_gem_detect_bit_6_swizzle(dev);
3919	init_waitqueue_head(&dev_priv->pending_flip_queue);	3919	init_waitqueue_head(&dev_priv->pending_flip_queue);
3920		3920
3921	dev_priv->mm.interruptible = true;	3921	dev_priv->mm.interruptible = true;
3922		3922
3923	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;	3923	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3924	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;	3924	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3925	register_shrinker(&dev_priv->mm.inactive_shrinker);	3925	register_shrinker(&dev_priv->mm.inactive_shrinker);
3926	}	3926	}
3927		3927
3928	/*	3928	/*
3929	* Create a physically contiguous memory object for this object	3929	* Create a physically contiguous memory object for this object
3930	* e.g. for cursor + overlay regs	3930	* e.g. for cursor + overlay regs
3931	*/	3931	*/
3932	static int i915_gem_init_phys_object(struct drm_device *dev,	3932	static int i915_gem_init_phys_object(struct drm_device *dev,
3933	int id, int size, int align)	3933	int id, int size, int align)
3934	{	3934	{
3935	drm_i915_private_t *dev_priv = dev->dev_private;	3935	drm_i915_private_t *dev_priv = dev->dev_private;
3936	struct drm_i915_gem_phys_object *phys_obj;	3936	struct drm_i915_gem_phys_object *phys_obj;
3937	int ret;	3937	int ret;
3938		3938
3939	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)	3939	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
3940	return 0;	3940	return 0;
3941		3941
3942	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);	3942	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
3943	if (!phys_obj)	3943	if (!phys_obj)
3944	return -ENOMEM;	3944	return -ENOMEM;
3945		3945
3946	phys_obj->id = id;	3946	phys_obj->id = id;
3947		3947
3948	phys_obj->handle = drm_pci_alloc(dev, size, align);	3948	phys_obj->handle = drm_pci_alloc(dev, size, align);
3949	if (!phys_obj->handle) {	3949	if (!phys_obj->handle) {
3950	ret = -ENOMEM;	3950	ret = -ENOMEM;
3951	goto kfree_obj;	3951	goto kfree_obj;
3952	}	3952	}
3953	#ifdef CONFIG_X86	3953	#ifdef CONFIG_X86
3954	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);	3954	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3955	#endif	3955	#endif
3956		3956
3957	dev_priv->mm.phys_objs[id - 1] = phys_obj;	3957	dev_priv->mm.phys_objs[id - 1] = phys_obj;
3958		3958
3959	return 0;	3959	return 0;
3960	kfree_obj:	3960	kfree_obj:
3961	kfree(phys_obj);	3961	kfree(phys_obj);
3962	return ret;	3962	return ret;
3963	}	3963	}
3964		3964
3965	static void i915_gem_free_phys_object(struct drm_device *dev, int id)	3965	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
3966	{	3966	{
3967	drm_i915_private_t *dev_priv = dev->dev_private;	3967	drm_i915_private_t *dev_priv = dev->dev_private;
3968	struct drm_i915_gem_phys_object *phys_obj;	3968	struct drm_i915_gem_phys_object *phys_obj;
3969		3969
3970	if (!dev_priv->mm.phys_objs[id - 1])	3970	if (!dev_priv->mm.phys_objs[id - 1])
3971	return;	3971	return;
3972		3972
3973	phys_obj = dev_priv->mm.phys_objs[id - 1];	3973	phys_obj = dev_priv->mm.phys_objs[id - 1];
3974	if (phys_obj->cur_obj) {	3974	if (phys_obj->cur_obj) {
3975	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);	3975	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3976	}	3976	}
3977		3977
3978	#ifdef CONFIG_X86	3978	#ifdef CONFIG_X86
3979	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);	3979	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3980	#endif	3980	#endif
3981	drm_pci_free(dev, phys_obj->handle);	3981	drm_pci_free(dev, phys_obj->handle);
3982	kfree(phys_obj);	3982	kfree(phys_obj);
3983	dev_priv->mm.phys_objs[id - 1] = NULL;	3983	dev_priv->mm.phys_objs[id - 1] = NULL;
3984	}	3984	}
3985		3985
3986	void i915_gem_free_all_phys_object(struct drm_device *dev)	3986	void i915_gem_free_all_phys_object(struct drm_device *dev)
3987	{	3987	{
3988	int i;	3988	int i;
3989		3989
3990	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)	3990	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3991	i915_gem_free_phys_object(dev, i);	3991	i915_gem_free_phys_object(dev, i);
3992	}	3992	}
3993		3993
3994	void i915_gem_detach_phys_object(struct drm_device *dev,	3994	void i915_gem_detach_phys_object(struct drm_device *dev,
3995	struct drm_i915_gem_object *obj)	3995	struct drm_i915_gem_object *obj)
3996	{	3996	{
3997	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	3997	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3998	char *vaddr;	3998	char *vaddr;
3999	int i;	3999	int i;
4000	int page_count;	4000	int page_count;
4001		4001
4002	if (!obj->phys_obj)	4002	if (!obj->phys_obj)
4003	return;	4003	return;
4004	vaddr = obj->phys_obj->handle->vaddr;	4004	vaddr = obj->phys_obj->handle->vaddr;
4005		4005
4006	page_count = obj->base.size / PAGE_SIZE;	4006	page_count = obj->base.size / PAGE_SIZE;
4007	for (i = 0; i < page_count; i++) {	4007	for (i = 0; i < page_count; i++) {
4008	struct page *page = shmem_read_mapping_page(mapping, i);	4008	struct page *page = shmem_read_mapping_page(mapping, i);
4009	if (!IS_ERR(page)) {	4009	if (!IS_ERR(page)) {
4010	char *dst = kmap_atomic(page);	4010	char *dst = kmap_atomic(page);
4011	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);	4011	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4012	kunmap_atomic(dst);	4012	kunmap_atomic(dst);
4013		4013
4014	drm_clflush_pages(&page, 1);	4014	drm_clflush_pages(&page, 1);
4015		4015
4016	set_page_dirty(page);	4016	set_page_dirty(page);
4017	mark_page_accessed(page);	4017	mark_page_accessed(page);
4018	page_cache_release(page);	4018	page_cache_release(page);
4019	}	4019	}
4020	}	4020	}
4021	intel_gtt_chipset_flush();	4021	intel_gtt_chipset_flush();
4022		4022
4023	obj->phys_obj->cur_obj = NULL;	4023	obj->phys_obj->cur_obj = NULL;
4024	obj->phys_obj = NULL;	4024	obj->phys_obj = NULL;
4025	}	4025	}
4026		4026
4027	int	4027	int
4028	i915_gem_attach_phys_object(struct drm_device *dev,	4028	i915_gem_attach_phys_object(struct drm_device *dev,
4029	struct drm_i915_gem_object *obj,	4029	struct drm_i915_gem_object *obj,
4030	int id,	4030	int id,
4031	int align)	4031	int align)
4032	{	4032	{
4033	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;	4033	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4034	drm_i915_private_t *dev_priv = dev->dev_private;	4034	drm_i915_private_t *dev_priv = dev->dev_private;
4035	int ret = 0;	4035	int ret = 0;
4036	int page_count;	4036	int page_count;
4037	int i;	4037	int i;
4038		4038
4039	if (id > I915_MAX_PHYS_OBJECT)	4039	if (id > I915_MAX_PHYS_OBJECT)
4040	return -EINVAL;	4040	return -EINVAL;
4041		4041
4042	if (obj->phys_obj) {	4042	if (obj->phys_obj) {
4043	if (obj->phys_obj->id == id)	4043	if (obj->phys_obj->id == id)
4044	return 0;	4044	return 0;
4045	i915_gem_detach_phys_object(dev, obj);	4045	i915_gem_detach_phys_object(dev, obj);
4046	}	4046	}
4047		4047
4048	/* create a new object */	4048	/* create a new object */
4049	if (!dev_priv->mm.phys_objs[id - 1]) {	4049	if (!dev_priv->mm.phys_objs[id - 1]) {
4050	ret = i915_gem_init_phys_object(dev, id,	4050	ret = i915_gem_init_phys_object(dev, id,
4051	obj->base.size, align);	4051	obj->base.size, align);
4052	if (ret) {	4052	if (ret) {
4053	DRM_ERROR("failed to init phys object %d size: %zu\n",	4053	DRM_ERROR("failed to init phys object %d size: %zu\n",
4054	id, obj->base.size);	4054	id, obj->base.size);
4055	return ret;	4055	return ret;
4056	}	4056	}
4057	}	4057	}
4058		4058
4059	/* bind to the object */	4059	/* bind to the object */
4060	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];	4060	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4061	obj->phys_obj->cur_obj = obj;	4061	obj->phys_obj->cur_obj = obj;
4062		4062
4063	page_count = obj->base.size / PAGE_SIZE;	4063	page_count = obj->base.size / PAGE_SIZE;
4064		4064
4065	for (i = 0; i < page_count; i++) {	4065	for (i = 0; i < page_count; i++) {
4066	struct page *page;	4066	struct page *page;
4067	char dst, src;	4067	char dst, src;
4068		4068
4069	page = shmem_read_mapping_page(mapping, i);	4069	page = shmem_read_mapping_page(mapping, i);
4070	if (IS_ERR(page))	4070	if (IS_ERR(page))
4071	return PTR_ERR(page);	4071	return PTR_ERR(page);
4072		4072
4073	src = kmap_atomic(page);	4073	src = kmap_atomic(page);
4074	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);	4074	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4075	memcpy(dst, src, PAGE_SIZE);	4075	memcpy(dst, src, PAGE_SIZE);
4076	kunmap_atomic(src);	4076	kunmap_atomic(src);
4077		4077
4078	mark_page_accessed(page);	4078	mark_page_accessed(page);
4079	page_cache_release(page);	4079	page_cache_release(page);
4080	}	4080	}
4081		4081
4082	return 0;	4082	return 0;
4083	}	4083	}
4084		4084
4085	static int	4085	static int
4086	i915_gem_phys_pwrite(struct drm_device *dev,	4086	i915_gem_phys_pwrite(struct drm_device *dev,
4087	struct drm_i915_gem_object *obj,	4087	struct drm_i915_gem_object *obj,
4088	struct drm_i915_gem_pwrite *args,	4088	struct drm_i915_gem_pwrite *args,
4089	struct drm_file *file_priv)	4089	struct drm_file *file_priv)
4090	{	4090	{
4091	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;	4091	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4092	char __user user_data = (char __user ) (uintptr_t) args->data_ptr;	4092	char __user user_data = (char __user ) (uintptr_t) args->data_ptr;
4093		4093
4094	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {	4094	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4095	unsigned long unwritten;	4095	unsigned long unwritten;
4096		4096
4097	/* The physical object once assigned is fixed for the lifetime	4097	/* The physical object once assigned is fixed for the lifetime
4098	* of the obj, so we can safely drop the lock and continue	4098	* of the obj, so we can safely drop the lock and continue
4099	* to access vaddr.	4099	* to access vaddr.
4100	*/	4100	*/
4101	mutex_unlock(&dev->struct_mutex);	4101	mutex_unlock(&dev->struct_mutex);
4102	unwritten = copy_from_user(vaddr, user_data, args->size);	4102	unwritten = copy_from_user(vaddr, user_data, args->size);
4103	mutex_lock(&dev->struct_mutex);	4103	mutex_lock(&dev->struct_mutex);
4104	if (unwritten)	4104	if (unwritten)
4105	return -EFAULT;	4105	return -EFAULT;
4106	}	4106	}
4107		4107
4108	intel_gtt_chipset_flush();	4108	intel_gtt_chipset_flush();
4109	return 0;	4109	return 0;
4110	}	4110	}
4111		4111
4112	void i915_gem_release(struct drm_device dev, struct drm_file file)	4112	void i915_gem_release(struct drm_device dev, struct drm_file file)
4113	{	4113	{
4114	struct drm_i915_file_private *file_priv = file->driver_priv;	4114	struct drm_i915_file_private *file_priv = file->driver_priv;
4115		4115
4116	/* Clean up our request list when the client is going away, so that	4116	/* Clean up our request list when the client is going away, so that
4117	* later retire_requests won't dereference our soon-to-be-gone	4117	* later retire_requests won't dereference our soon-to-be-gone
4118	* file_priv.	4118	* file_priv.
4119	*/	4119	*/
4120	spin_lock(&file_priv->mm.lock);	4120	spin_lock(&file_priv->mm.lock);
4121	while (!list_empty(&file_priv->mm.request_list)) {	4121	while (!list_empty(&file_priv->mm.request_list)) {
4122	struct drm_i915_gem_request *request;	4122	struct drm_i915_gem_request *request;
4123		4123
4124	request = list_first_entry(&file_priv->mm.request_list,	4124	request = list_first_entry(&file_priv->mm.request_list,
4125	struct drm_i915_gem_request,	4125	struct drm_i915_gem_request,
4126	client_list);	4126	client_list);
4127	list_del(&request->client_list);	4127	list_del(&request->client_list);
4128	request->file_priv = NULL;	4128	request->file_priv = NULL;
4129	}	4129	}
4130	spin_unlock(&file_priv->mm.lock);	4130	spin_unlock(&file_priv->mm.lock);
4131	}	4131	}
4132		4132
4133	static int	4133	static int
4134	i915_gpu_is_active(struct drm_device *dev)	4134	i915_gpu_is_active(struct drm_device *dev)
4135	{	4135	{
4136	drm_i915_private_t *dev_priv = dev->dev_private;	4136	drm_i915_private_t *dev_priv = dev->dev_private;
4137	int lists_empty;	4137	int lists_empty;
4138		4138
4139	lists_empty = list_empty(&dev_priv->mm.flushing_list) &&	4139	lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4140	list_empty(&dev_priv->mm.active_list);	4140	list_empty(&dev_priv->mm.active_list);
4141		4141
4142	return !lists_empty;	4142	return !lists_empty;
4143	}	4143	}
4144		4144
4145	static int	4145	static int
4146	i915_gem_inactive_shrink(struct shrinker shrinker, struct shrink_control sc)	4146	i915_gem_inactive_shrink(struct shrinker shrinker, struct shrink_control sc)
4147	{	4147	{
4148	struct drm_i915_private *dev_priv =	4148	struct drm_i915_private *dev_priv =
4149	container_of(shrinker,	4149	container_of(shrinker,
4150	struct drm_i915_private,	4150	struct drm_i915_private,
4151	mm.inactive_shrinker);	4151	mm.inactive_shrinker);
4152	struct drm_device *dev = dev_priv->dev;	4152	struct drm_device *dev = dev_priv->dev;
4153	struct drm_i915_gem_object obj, next;	4153	struct drm_i915_gem_object obj, next;
4154	int nr_to_scan = sc->nr_to_scan;	4154	int nr_to_scan = sc->nr_to_scan;
4155	int cnt;	4155	int cnt;
4156		4156
4157	if (!mutex_trylock(&dev->struct_mutex))	4157	if (!mutex_trylock(&dev->struct_mutex))
4158	return 0;	4158	return 0;
4159		4159
4160	/* "fast-path" to count number of available objects */	4160	/* "fast-path" to count number of available objects */
4161	if (nr_to_scan == 0) {	4161	if (nr_to_scan == 0) {
4162	cnt = 0;	4162	cnt = 0;
4163	list_for_each_entry(obj,	4163	list_for_each_entry(obj,
4164	&dev_priv->mm.inactive_list,	4164	&dev_priv->mm.inactive_list,
4165	mm_list)	4165	mm_list)
4166	cnt++;	4166	cnt++;
4167	mutex_unlock(&dev->struct_mutex);	4167	mutex_unlock(&dev->struct_mutex);
4168	return cnt / 100 * sysctl_vfs_cache_pressure;	4168	return cnt / 100 * sysctl_vfs_cache_pressure;
4169	}	4169	}
4170		4170
4171	rescan:	4171	rescan:
4172	/* first scan for clean buffers */	4172	/* first scan for clean buffers */
4173	i915_gem_retire_requests(dev);	4173	i915_gem_retire_requests(dev);
4174		4174
4175	list_for_each_entry_safe(obj, next,	4175	list_for_each_entry_safe(obj, next,
4176	&dev_priv->mm.inactive_list,	4176	&dev_priv->mm.inactive_list,
4177	mm_list) {	4177	mm_list) {
4178	if (i915_gem_object_is_purgeable(obj)) {	4178	if (i915_gem_object_is_purgeable(obj)) {
4179	if (i915_gem_object_unbind(obj) == 0 &&	4179	if (i915_gem_object_unbind(obj) == 0 &&
4180	--nr_to_scan == 0)	4180	--nr_to_scan == 0)
4181	break;	4181	break;
4182	}	4182	}
4183	}	4183	}
4184		4184
4185	/* second pass, evict/count anything still on the inactive list */	4185	/* second pass, evict/count anything still on the inactive list */
4186	cnt = 0;	4186	cnt = 0;
4187	list_for_each_entry_safe(obj, next,	4187	list_for_each_entry_safe(obj, next,
4188	&dev_priv->mm.inactive_list,	4188	&dev_priv->mm.inactive_list,
4189	mm_list) {	4189	mm_list) {
4190	if (nr_to_scan &&	4190	if (nr_to_scan &&
4191	i915_gem_object_unbind(obj) == 0)	4191	i915_gem_object_unbind(obj) == 0)
4192	nr_to_scan--;	4192	nr_to_scan--;
4193	else	4193	else
4194	cnt++;	4194	cnt++;
4195	}	4195	}
4196		4196
4197	if (nr_to_scan && i915_gpu_is_active(dev)) {	4197	if (nr_to_scan && i915_gpu_is_active(dev)) {
4198	/*	4198	/*
4199	* We are desperate for pages, so as a last resort, wait	4199	* We are desperate for pages, so as a last resort, wait
4200	* for the GPU to finish and discard whatever we can.	4200	* for the GPU to finish and discard whatever we can.
4201	* This has a dramatic impact to reduce the number of	4201	* This has a dramatic impact to reduce the number of
4202	* OOM-killer events whilst running the GPU aggressively.	4202	* OOM-killer events whilst running the GPU aggressively.
4203	*/	4203	*/
4204	if (i915_gpu_idle(dev) == 0)	4204	if (i915_gpu_idle(dev) == 0)
4205	goto rescan;	4205	goto rescan;
4206	}	4206	}
4207	mutex_unlock(&dev->struct_mutex);	4207	mutex_unlock(&dev->struct_mutex);
4208	return cnt / 100 * sysctl_vfs_cache_pressure;	4208	return cnt / 100 * sysctl_vfs_cache_pressure;
4209	}	4209	}
4210		4210

include/drm/i915_drm.h

Diff comments View file @ 3d29b84

 /*
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * without limitation the rights to use, copy, modify, merge, publish,
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
 #ifndef _I915_DRM_H_
 #define _I915_DRM_H_
 #include "drm.h"
 /* Please note that modifications to all structs defined here are
  * subject to backwards-compatibility constraints.
  */
 #ifdef __KERNEL__
 /* For use by IPS driver */
 extern unsigned long i915_read_mch_val(void);
 extern bool i915_gpu_raise(void);
 extern bool i915_gpu_lower(void);
 extern bool i915_gpu_busy(void);
 extern bool i915_gpu_turbo_disable(void);
 #endif
 /* Each region is a minimum of 16k, and there are at most 255 of them.
  */
 #define I915_NR_TEX_REGIONS 255	/* table size 2k - maximum due to use
 				 * of chars for next/prev indices */
 #define I915_LOG_MIN_TEX_REGION_SIZE 14
 typedef struct _drm_i915_init {
 	enum {
 		I915_INIT_DMA = 0x01,
 		I915_CLEANUP_DMA = 0x02,
 		I915_RESUME_DMA = 0x03
 	} func;
 	unsigned int mmio_offset;
 	int sarea_priv_offset;
 	unsigned int ring_start;
 	unsigned int ring_end;
 	unsigned int ring_size;
 	unsigned int front_offset;
 	unsigned int back_offset;
 	unsigned int depth_offset;
 	unsigned int w;
 	unsigned int h;
 	unsigned int pitch;
 	unsigned int pitch_bits;
 	unsigned int back_pitch;
 	unsigned int depth_pitch;
 	unsigned int cpp;
 	unsigned int chipset;
 } drm_i915_init_t;
 typedef struct _drm_i915_sarea {
 	struct drm_tex_region texList[I915_NR_TEX_REGIONS + 1];
 	int last_upload;	/* last time texture was uploaded */
 	int last_enqueue;	/* last time a buffer was enqueued */
 	int last_dispatch;	/* age of the most recently dispatched buffer */
 	int ctxOwner;		/* last context to upload state */
 	int texAge;
 	int pf_enabled;		/* is pageflipping allowed? */
 	int pf_active;
 	int pf_current_page;	/* which buffer is being displayed? */
 	int perf_boxes;		/* performance boxes to be displayed */
 	int width, height;      /* screen size in pixels */
 	drm_handle_t front_handle;
 	int front_offset;
 	int front_size;
 	drm_handle_t back_handle;
 	int back_offset;
 	int back_size;
 	drm_handle_t depth_handle;
 	int depth_offset;
 	int depth_size;
 	drm_handle_t tex_handle;
 	int tex_offset;
 	int tex_size;
 	int log_tex_granularity;
 	int pitch;
 	int rotation;           /* 0, 90, 180 or 270 */
 	int rotated_offset;
 	int rotated_size;
 	int rotated_pitch;
 	int virtualX, virtualY;
 	unsigned int front_tiled;
 	unsigned int back_tiled;
 	unsigned int depth_tiled;
 	unsigned int rotated_tiled;
 	unsigned int rotated2_tiled;
 	int pipeA_x;
 	int pipeA_y;
 	int pipeA_w;
 	int pipeA_h;
 	int pipeB_x;
 	int pipeB_y;
 	int pipeB_w;
 	int pipeB_h;
 	/* fill out some space for old userspace triple buffer */
 	drm_handle_t unused_handle;
 	__u32 unused1, unused2, unused3;
 	/* buffer object handles for static buffers. May change
 	 * over the lifetime of the client.
 	 */
 	__u32 front_bo_handle;
 	__u32 back_bo_handle;
 	__u32 unused_bo_handle;
 	__u32 depth_bo_handle;
 } drm_i915_sarea_t;
 /* due to userspace building against these headers we need some compat here */
 #define planeA_x pipeA_x
 #define planeA_y pipeA_y
 #define planeA_w pipeA_w
 #define planeA_h pipeA_h
 #define planeB_x pipeB_x
 #define planeB_y pipeB_y
 #define planeB_w pipeB_w
 #define planeB_h pipeB_h
 /* Flags for perf_boxes
  */
 #define I915_BOX_RING_EMPTY    0x1
 #define I915_BOX_FLIP          0x2
 #define I915_BOX_WAIT          0x4
 #define I915_BOX_TEXTURE_LOAD  0x8
 #define I915_BOX_LOST_CONTEXT  0x10
 /* I915 specific ioctls
  * The device specific ioctl range is 0x40 to 0x79.
  */
 #define DRM_I915_INIT		0x00
 #define DRM_I915_FLUSH		0x01
 #define DRM_I915_FLIP		0x02
 #define DRM_I915_BATCHBUFFER	0x03
 #define DRM_I915_IRQ_EMIT	0x04
 #define DRM_I915_IRQ_WAIT	0x05
 #define DRM_I915_GETPARAM	0x06
 #define DRM_I915_SETPARAM	0x07
 #define DRM_I915_ALLOC		0x08
 #define DRM_I915_FREE		0x09
 #define DRM_I915_INIT_HEAP	0x0a
 #define DRM_I915_CMDBUFFER	0x0b
 #define DRM_I915_DESTROY_HEAP	0x0c
 #define DRM_I915_SET_VBLANK_PIPE	0x0d
 #define DRM_I915_GET_VBLANK_PIPE	0x0e
 #define DRM_I915_VBLANK_SWAP	0x0f
 #define DRM_I915_HWS_ADDR	0x11
 #define DRM_I915_GEM_INIT	0x13
 #define DRM_I915_GEM_EXECBUFFER	0x14
 #define DRM_I915_GEM_PIN	0x15
 #define DRM_I915_GEM_UNPIN	0x16
 #define DRM_I915_GEM_BUSY	0x17
 #define DRM_I915_GEM_THROTTLE	0x18
 #define DRM_I915_GEM_ENTERVT	0x19
 #define DRM_I915_GEM_LEAVEVT	0x1a
 #define DRM_I915_GEM_CREATE	0x1b
 #define DRM_I915_GEM_PREAD	0x1c
 #define DRM_I915_GEM_PWRITE	0x1d
 #define DRM_I915_GEM_MMAP	0x1e
 #define DRM_I915_GEM_SET_DOMAIN	0x1f
 #define DRM_I915_GEM_SW_FINISH	0x20
 #define DRM_I915_GEM_SET_TILING	0x21
 #define DRM_I915_GEM_GET_TILING	0x22
 #define DRM_I915_GEM_GET_APERTURE 0x23
 #define DRM_I915_GEM_MMAP_GTT	0x24
 #define DRM_I915_GET_PIPE_FROM_CRTC_ID	0x25
 #define DRM_I915_GEM_MADVISE	0x26
 #define DRM_I915_OVERLAY_PUT_IMAGE	0x27
 #define DRM_I915_OVERLAY_ATTRS	0x28
 #define DRM_I915_GEM_EXECBUFFER2	0x29
 #define DRM_I915_GET_SPRITE_COLORKEY	0x2a
 #define DRM_I915_SET_SPRITE_COLORKEY	0x2b
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
 #define DRM_IOCTL_I915_FLIP		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLIP)
 #define DRM_IOCTL_I915_BATCHBUFFER	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_BATCHBUFFER, drm_i915_batchbuffer_t)
 #define DRM_IOCTL_I915_IRQ_EMIT         DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_IRQ_EMIT, drm_i915_irq_emit_t)
 #define DRM_IOCTL_I915_IRQ_WAIT         DRM_IOW( DRM_COMMAND_BASE + DRM_I915_IRQ_WAIT, drm_i915_irq_wait_t)
 #define DRM_IOCTL_I915_GETPARAM         DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GETPARAM, drm_i915_getparam_t)
 #define DRM_IOCTL_I915_SETPARAM         DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SETPARAM, drm_i915_setparam_t)
 #define DRM_IOCTL_I915_ALLOC            DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_ALLOC, drm_i915_mem_alloc_t)
 #define DRM_IOCTL_I915_FREE             DRM_IOW( DRM_COMMAND_BASE + DRM_I915_FREE, drm_i915_mem_free_t)
 #define DRM_IOCTL_I915_INIT_HEAP        DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT_HEAP, drm_i915_mem_init_heap_t)
 #define DRM_IOCTL_I915_CMDBUFFER	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_CMDBUFFER, drm_i915_cmdbuffer_t)
 #define DRM_IOCTL_I915_DESTROY_HEAP	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_DESTROY_HEAP, drm_i915_mem_destroy_heap_t)
 #define DRM_IOCTL_I915_SET_VBLANK_PIPE	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
 #define DRM_IOCTL_I915_GET_VBLANK_PIPE	DRM_IOR( DRM_COMMAND_BASE + DRM_I915_GET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
 #define DRM_IOCTL_I915_VBLANK_SWAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_VBLANK_SWAP, drm_i915_vblank_swap_t)
 #define DRM_IOCTL_I915_HWS_ADDR		DRM_IOW(DRM_COMMAND_BASE + DRM_I915_HWS_ADDR, struct drm_i915_gem_init)
 #define DRM_IOCTL_I915_GEM_INIT		DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_INIT, struct drm_i915_gem_init)
 #define DRM_IOCTL_I915_GEM_EXECBUFFER	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_EXECBUFFER, struct drm_i915_gem_execbuffer)
 #define DRM_IOCTL_I915_GEM_EXECBUFFER2	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_EXECBUFFER2, struct drm_i915_gem_execbuffer2)
 #define DRM_IOCTL_I915_GEM_PIN		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_PIN, struct drm_i915_gem_pin)
 #define DRM_IOCTL_I915_GEM_UNPIN	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_UNPIN, struct drm_i915_gem_unpin)
 #define DRM_IOCTL_I915_GEM_BUSY		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_BUSY, struct drm_i915_gem_busy)
 #define DRM_IOCTL_I915_GEM_THROTTLE	DRM_IO ( DRM_COMMAND_BASE + DRM_I915_GEM_THROTTLE)
 #define DRM_IOCTL_I915_GEM_ENTERVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_ENTERVT)
 #define DRM_IOCTL_I915_GEM_LEAVEVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_LEAVEVT)
 #define DRM_IOCTL_I915_GEM_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_CREATE, struct drm_i915_gem_create)
 #define DRM_IOCTL_I915_GEM_PREAD	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PREAD, struct drm_i915_gem_pread)
 #define DRM_IOCTL_I915_GEM_PWRITE	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite)
 #define DRM_IOCTL_I915_GEM_MMAP		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap)
 #define DRM_IOCTL_I915_GEM_MMAP_GTT	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_gtt)
 #define DRM_IOCTL_I915_GEM_SET_DOMAIN	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain)
 #define DRM_IOCTL_I915_GEM_SW_FINISH	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish)
 #define DRM_IOCTL_I915_GEM_SET_TILING	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_SET_TILING, struct drm_i915_gem_set_tiling)
 #define DRM_IOCTL_I915_GEM_GET_TILING	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_TILING, struct drm_i915_gem_get_tiling)
 #define DRM_IOCTL_I915_GEM_GET_APERTURE	DRM_IOR  (DRM_COMMAND_BASE + DRM_I915_GEM_GET_APERTURE, struct drm_i915_gem_get_aperture)
 #define DRM_IOCTL_I915_GET_PIPE_FROM_CRTC_ID DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GET_PIPE_FROM_CRTC_ID, struct drm_i915_get_pipe_from_crtc_id)
 #define DRM_IOCTL_I915_GEM_MADVISE	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MADVISE, struct drm_i915_gem_madvise)
 #define DRM_IOCTL_I915_OVERLAY_PUT_IMAGE	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_OVERLAY_PUT_IMAGE, struct drm_intel_overlay_put_image)
 #define DRM_IOCTL_I915_OVERLAY_ATTRS	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_OVERLAY_ATTRS, struct drm_intel_overlay_attrs)
 #define DRM_IOCTL_I915_SET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_SET_SPRITE_COLORKEY, struct drm_intel_sprite_colorkey)
 #define DRM_IOCTL_I915_GET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_SET_SPRITE_COLORKEY, struct drm_intel_sprite_colorkey)
 /* Allow drivers to submit batchbuffers directly to hardware, relying
  * on the security mechanisms provided by hardware.
  */
 typedef struct drm_i915_batchbuffer {
 	int start;		/* agp offset */
 	int used;		/* nr bytes in use */
 	int DR1;		/* hw flags for GFX_OP_DRAWRECT_INFO */
 	int DR4;		/* window origin for GFX_OP_DRAWRECT_INFO */
 	int num_cliprects;	/* mulitpass with multiple cliprects? */
 	struct drm_clip_rect __user *cliprects;	/* pointer to userspace cliprects */
 } drm_i915_batchbuffer_t;
 /* As above, but pass a pointer to userspace buffer which can be
  * validated by the kernel prior to sending to hardware.
  */
 typedef struct _drm_i915_cmdbuffer {
 	char __user *buf;	/* pointer to userspace command buffer */
 	int sz;			/* nr bytes in buf */
 	int DR1;		/* hw flags for GFX_OP_DRAWRECT_INFO */
 	int DR4;		/* window origin for GFX_OP_DRAWRECT_INFO */
 	int num_cliprects;	/* mulitpass with multiple cliprects? */
 	struct drm_clip_rect __user *cliprects;	/* pointer to userspace cliprects */
 } drm_i915_cmdbuffer_t;
 /* Userspace can request & wait on irq's:
  */
 typedef struct drm_i915_irq_emit {
 	int __user *irq_seq;
 } drm_i915_irq_emit_t;
 typedef struct drm_i915_irq_wait {
 	int irq_seq;
 } drm_i915_irq_wait_t;
 /* Ioctl to query kernel params:
  */
 #define I915_PARAM_IRQ_ACTIVE            1
 #define I915_PARAM_ALLOW_BATCHBUFFER     2
 #define I915_PARAM_LAST_DISPATCH         3
 #define I915_PARAM_CHIPSET_ID            4
 #define I915_PARAM_HAS_GEM               5
 #define I915_PARAM_NUM_FENCES_AVAIL      6
 #define I915_PARAM_HAS_OVERLAY           7
 #define I915_PARAM_HAS_PAGEFLIPPING	 8
 #define I915_PARAM_HAS_EXECBUF2          9
 #define I915_PARAM_HAS_BSD		 10
 #define I915_PARAM_HAS_BLT		 11
 #define I915_PARAM_HAS_RELAXED_FENCING	 12
 #define I915_PARAM_HAS_COHERENT_RINGS	 13
 #define I915_PARAM_HAS_EXEC_CONSTANTS	 14
 #define I915_PARAM_HAS_RELAXED_DELTA	 15
 #define I915_PARAM_HAS_GEN7_SOL_RESET	 16
+#define I915_PARAM_HAS_LLC     	 17
 typedef struct drm_i915_getparam {
 	int param;
 	int __user *value;
 } drm_i915_getparam_t;
 /* Ioctl to set kernel params:
  */
 #define I915_SETPARAM_USE_MI_BATCHBUFFER_START            1
 #define I915_SETPARAM_TEX_LRU_LOG_GRANULARITY             2
 #define I915_SETPARAM_ALLOW_BATCHBUFFER                   3
 #define I915_SETPARAM_NUM_USED_FENCES                     4
 typedef struct drm_i915_setparam {
 	int param;
 	int value;
 } drm_i915_setparam_t;
 /* A memory manager for regions of shared memory:
  */
 #define I915_MEM_REGION_AGP 1
 typedef struct drm_i915_mem_alloc {
 	int region;
 	int alignment;
 	int size;
 	int __user *region_offset;	/* offset from start of fb or agp */
 } drm_i915_mem_alloc_t;
 typedef struct drm_i915_mem_free {
 	int region;
 	int region_offset;
 } drm_i915_mem_free_t;
 typedef struct drm_i915_mem_init_heap {
 	int region;
 	int size;
 	int start;
 } drm_i915_mem_init_heap_t;
 /* Allow memory manager to be torn down and re-initialized (eg on
  * rotate):
  */
 typedef struct drm_i915_mem_destroy_heap {
 	int region;
 } drm_i915_mem_destroy_heap_t;
 /* Allow X server to configure which pipes to monitor for vblank signals
  */
 #define	DRM_I915_VBLANK_PIPE_A	1
 #define	DRM_I915_VBLANK_PIPE_B	2
 typedef struct drm_i915_vblank_pipe {
 	int pipe;
 } drm_i915_vblank_pipe_t;
 /* Schedule buffer swap at given vertical blank:
  */
 typedef struct drm_i915_vblank_swap {
 	drm_drawable_t drawable;
 	enum drm_vblank_seq_type seqtype;
 	unsigned int sequence;
 } drm_i915_vblank_swap_t;
 typedef struct drm_i915_hws_addr {
 	__u64 addr;
 } drm_i915_hws_addr_t;
 struct drm_i915_gem_init {
 	/**
 	 * Beginning offset in the GTT to be managed by the DRM memory
 	 * manager.
 	 */
 	__u64 gtt_start;
 	/**
 	 * Ending offset in the GTT to be managed by the DRM memory
 	 * manager.
 	 */
 	__u64 gtt_end;
 };
 struct drm_i915_gem_create {
 	/**
 	 * Requested size for the object.
 	 *
 	 * The (page-aligned) allocated size for the object will be returned.
 	 */
 	__u64 size;
 	/**
 	 * Returned handle for the object.
 	 *
 	 * Object handles are nonzero.
 	 */
 	__u32 handle;
 	__u32 pad;
 };
 struct drm_i915_gem_pread {
 	/** Handle for the object being read. */
 	__u32 handle;
 	__u32 pad;
 	/** Offset into the object to read from */
 	__u64 offset;
 	/** Length of data to read */
 	__u64 size;
 	/**
 	 * Pointer to write the data into.
 	 *
 	 * This is a fixed-size type for 32/64 compatibility.
 	 */
 	__u64 data_ptr;
 };
 struct drm_i915_gem_pwrite {
 	/** Handle for the object being written to. */
 	__u32 handle;
 	__u32 pad;
 	/** Offset into the object to write to */
 	__u64 offset;
 	/** Length of data to write */
 	__u64 size;
 	/**
 	 * Pointer to read the data from.
 	 *
 	 * This is a fixed-size type for 32/64 compatibility.
 	 */
 	__u64 data_ptr;
 };
 struct drm_i915_gem_mmap {
 	/** Handle for the object being mapped. */
 	__u32 handle;
 	__u32 pad;
 	/** Offset in the object to map. */
 	__u64 offset;
 	/**
 	 * Length of data to map.
 	 *
 	 * The value will be page-aligned.
 	 */
 	__u64 size;
 	/**
 	 * Returned pointer the data was mapped at.
 	 *
 	 * This is a fixed-size type for 32/64 compatibility.
 	 */
 	__u64 addr_ptr;
 };
 struct drm_i915_gem_mmap_gtt {
 	/** Handle for the object being mapped. */
 	__u32 handle;
 	__u32 pad;
 	/**
 	 * Fake offset to use for subsequent mmap call
 	 *
 	 * This is a fixed-size type for 32/64 compatibility.
 	 */
 	__u64 offset;
 };
 struct drm_i915_gem_set_domain {
 	/** Handle for the object */
 	__u32 handle;
 	/** New read domains */
 	__u32 read_domains;
 	/** New write domain */
 	__u32 write_domain;
 };
 struct drm_i915_gem_sw_finish {
 	/** Handle for the object */
 	__u32 handle;
 };
 struct drm_i915_gem_relocation_entry {
 	/**
 	 * Handle of the buffer being pointed to by this relocation entry.
 	 *
 	 * It's appealing to make this be an index into the mm_validate_entry
 	 * list to refer to the buffer, but this allows the driver to create
 	 * a relocation list for state buffers and not re-write it per
 	 * exec using the buffer.
 	 */
 	__u32 target_handle;
 	/**
 	 * Value to be added to the offset of the target buffer to make up
 	 * the relocation entry.
 	 */
 	__u32 delta;
 	/** Offset in the buffer the relocation entry will be written into */
 	__u64 offset;
 	/**
 	 * Offset value of the target buffer that the relocation entry was last
 	 * written as.
 	 *
 	 * If the buffer has the same offset as last time, we can skip syncing
 	 * and writing the relocation.  This value is written back out by
 	 * the execbuffer ioctl when the relocation is written.
 	 */
 	__u64 presumed_offset;
 	/**
 	 * Target memory domains read by this operation.
 	 */
 	__u32 read_domains;
 	/**
 	 * Target memory domains written by this operation.
 	 *
 	 * Note that only one domain may be written by the whole
 	 * execbuffer operation, so that where there are conflicts,
 	 * the application will get -EINVAL back.
 	 */
 	__u32 write_domain;
 };
 /** @{
  * Intel memory domains
  *
  * Most of these just align with the various caches in
  * the system and are used to flush and invalidate as
  * objects end up cached in different domains.
  */
 /** CPU cache */
 #define I915_GEM_DOMAIN_CPU		0x00000001
 /** Render cache, used by 2D and 3D drawing */
 #define I915_GEM_DOMAIN_RENDER		0x00000002
 /** Sampler cache, used by texture engine */
 #define I915_GEM_DOMAIN_SAMPLER		0x00000004
 /** Command queue, used to load batch buffers */
 #define I915_GEM_DOMAIN_COMMAND		0x00000008
 /** Instruction cache, used by shader programs */
 #define I915_GEM_DOMAIN_INSTRUCTION	0x00000010
 /** Vertex address cache */
 #define I915_GEM_DOMAIN_VERTEX		0x00000020
 /** GTT domain - aperture and scanout */
 #define I915_GEM_DOMAIN_GTT		0x00000040
 /** @} */
 struct drm_i915_gem_exec_object {
 	/**
 	 * User's handle for a buffer to be bound into the GTT for this
 	 * operation.
 	 */
 	__u32 handle;
 	/** Number of relocations to be performed on this buffer */
 	__u32 relocation_count;
 	/**
 	 * Pointer to array of struct drm_i915_gem_relocation_entry containing
 	 * the relocations to be performed in this buffer.
 	 */
 	__u64 relocs_ptr;
 	/** Required alignment in graphics aperture */
 	__u64 alignment;
 	/**
 	 * Returned value of the updated offset of the object, for future
 	 * presumed_offset writes.
 	 */
 	__u64 offset;
 };
 struct drm_i915_gem_execbuffer {
 	/**
 	 * List of buffers to be validated with their relocations to be
 	 * performend on them.
 	 *
 	 * This is a pointer to an array of struct drm_i915_gem_validate_entry.
 	 *
 	 * These buffers must be listed in an order such that all relocations
 	 * a buffer is performing refer to buffers that have already appeared
 	 * in the validate list.
 	 */
 	__u64 buffers_ptr;
 	__u32 buffer_count;
 	/** Offset in the batchbuffer to start execution from. */
 	__u32 batch_start_offset;
 	/** Bytes used in batchbuffer from batch_start_offset */
 	__u32 batch_len;
 	__u32 DR1;
 	__u32 DR4;
 	__u32 num_cliprects;
 	/** This is a struct drm_clip_rect *cliprects */
 	__u64 cliprects_ptr;
 };
 struct drm_i915_gem_exec_object2 {
 	/**
 	 * User's handle for a buffer to be bound into the GTT for this
 	 * operation.
 	 */
 	__u32 handle;
 	/** Number of relocations to be performed on this buffer */
 	__u32 relocation_count;
 	/**
 	 * Pointer to array of struct drm_i915_gem_relocation_entry containing
 	 * the relocations to be performed in this buffer.
 	 */
 	__u64 relocs_ptr;
 	/** Required alignment in graphics aperture */
 	__u64 alignment;
 	/**
 	 * Returned value of the updated offset of the object, for future
 	 * presumed_offset writes.
 	 */
 	__u64 offset;
 #define EXEC_OBJECT_NEEDS_FENCE (1<<0)
 	__u64 flags;
 	__u64 rsvd1;
 	__u64 rsvd2;
 };
 struct drm_i915_gem_execbuffer2 {
 	/**
 	 * List of gem_exec_object2 structs
 	 */
 	__u64 buffers_ptr;
 	__u32 buffer_count;
 	/** Offset in the batchbuffer to start execution from. */
 	__u32 batch_start_offset;
 	/** Bytes used in batchbuffer from batch_start_offset */
 	__u32 batch_len;
 	__u32 DR1;
 	__u32 DR4;
 	__u32 num_cliprects;
 	/** This is a struct drm_clip_rect *cliprects */
 	__u64 cliprects_ptr;
 #define I915_EXEC_RING_MASK              (7<<0)
 #define I915_EXEC_DEFAULT                (0<<0)
 #define I915_EXEC_RENDER                 (1<<0)
 #define I915_EXEC_BSD                    (2<<0)
 #define I915_EXEC_BLT                    (3<<0)
 /* Used for switching the constants addressing mode on gen4+ RENDER ring.
  * Gen6+ only supports relative addressing to dynamic state (default) and
  * absolute addressing.
  *
  * These flags are ignored for the BSD and BLT rings.
  */
 #define I915_EXEC_CONSTANTS_MASK 	(3<<6)
 #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
 #define I915_EXEC_CONSTANTS_ABSOLUTE 	(1<<6)
 #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
 	__u64 flags;
 	__u64 rsvd1;
 	__u64 rsvd2;
 };
 /** Resets the SO write offset registers for transform feedback on gen7. */
 #define I915_EXEC_GEN7_SOL_RESET	(1<<8)
 struct drm_i915_gem_pin {
 	/** Handle of the buffer to be pinned. */
 	__u32 handle;
 	__u32 pad;
 	/** alignment required within the aperture */
 	__u64 alignment;
 	/** Returned GTT offset of the buffer. */
 	__u64 offset;
 };
 struct drm_i915_gem_unpin {
 	/** Handle of the buffer to be unpinned. */
 	__u32 handle;
 	__u32 pad;
 };
 struct drm_i915_gem_busy {
 	/** Handle of the buffer to check for busy */
 	__u32 handle;
 	/** Return busy status (1 if busy, 0 if idle) */
 	__u32 busy;
 };
 #define I915_TILING_NONE	0
 #define I915_TILING_X		1
 #define I915_TILING_Y		2
 #define I915_BIT_6_SWIZZLE_NONE		0
 #define I915_BIT_6_SWIZZLE_9		1
 #define I915_BIT_6_SWIZZLE_9_10		2
 #define I915_BIT_6_SWIZZLE_9_11		3
 #define I915_BIT_6_SWIZZLE_9_10_11	4
 /* Not seen by userland */
 #define I915_BIT_6_SWIZZLE_UNKNOWN	5
 /* Seen by userland. */
 #define I915_BIT_6_SWIZZLE_9_17		6
 #define I915_BIT_6_SWIZZLE_9_10_17	7
 struct drm_i915_gem_set_tiling {
 	/** Handle of the buffer to have its tiling state updated */
 	__u32 handle;
 	/**
 	 * Tiling mode for the object (I915_TILING_NONE, I915_TILING_X,
 	 * I915_TILING_Y).
 	 *
 	 * This value is to be set on request, and will be updated by the
 	 * kernel on successful return with the actual chosen tiling layout.
 	 *
 	 * The tiling mode may be demoted to I915_TILING_NONE when the system
 	 * has bit 6 swizzling that can't be managed correctly by GEM.
 	 *
 	 * Buffer contents become undefined when changing tiling_mode.
 	 */
 	__u32 tiling_mode;
 	/**
 	 * Stride in bytes for the object when in I915_TILING_X or
 	 * I915_TILING_Y.
 	 */
 	__u32 stride;
 	/**
 	 * Returned address bit 6 swizzling required for CPU access through
 	 * mmap mapping.
 	 */
 	__u32 swizzle_mode;
 };
 struct drm_i915_gem_get_tiling {
 	/** Handle of the buffer to get tiling state for. */
 	__u32 handle;
 	/**
 	 * Current tiling mode for the object (I915_TILING_NONE, I915_TILING_X,
 	 * I915_TILING_Y).
 	 */
 	__u32 tiling_mode;
 	/**
 	 * Returned address bit 6 swizzling required for CPU access through
 	 * mmap mapping.
 	 */
 	__u32 swizzle_mode;
 };
 struct drm_i915_gem_get_aperture {
 	/** Total size of the aperture used by i915_gem_execbuffer, in bytes */
 	__u64 aper_size;
 	/**
 	 * Available space in the aperture used by i915_gem_execbuffer, in
 	 * bytes
 	 */
 	__u64 aper_available_size;
 };
 struct drm_i915_get_pipe_from_crtc_id {
 	/** ID of CRTC being requested **/
 	__u32 crtc_id;
 	/** pipe of requested CRTC **/
 	__u32 pipe;
 };
 #define I915_MADV_WILLNEED 0
 #define I915_MADV_DONTNEED 1
 #define __I915_MADV_PURGED 2 /* internal state */
 struct drm_i915_gem_madvise {
 	/** Handle of the buffer to change the backing store advice */
 	__u32 handle;
 	/* Advice: either the buffer will be needed again in the near future,
 	 *         or wont be and could be discarded under memory pressure.
 	 */
 	__u32 madv;
 	/** Whether the backing store still exists. */
 	__u32 retained;
 };
 /* flags */
 #define I915_OVERLAY_TYPE_MASK 		0xff
 #define I915_OVERLAY_YUV_PLANAR 	0x01
 #define I915_OVERLAY_YUV_PACKED 	0x02
 #define I915_OVERLAY_RGB		0x03
 #define I915_OVERLAY_DEPTH_MASK		0xff00
 #define I915_OVERLAY_RGB24		0x1000
 #define I915_OVERLAY_RGB16		0x2000
 #define I915_OVERLAY_RGB15		0x3000
 #define I915_OVERLAY_YUV422		0x0100
 #define I915_OVERLAY_YUV411		0x0200
 #define I915_OVERLAY_YUV420		0x0300
 #define I915_OVERLAY_YUV410		0x0400
 #define I915_OVERLAY_SWAP_MASK		0xff0000
 #define I915_OVERLAY_NO_SWAP		0x000000
 #define I915_OVERLAY_UV_SWAP		0x010000
 #define I915_OVERLAY_Y_SWAP		0x020000
 #define I915_OVERLAY_Y_AND_UV_SWAP	0x030000
 #define I915_OVERLAY_FLAGS_MASK		0xff000000
 #define I915_OVERLAY_ENABLE		0x01000000
 struct drm_intel_overlay_put_image {
 	/* various flags and src format description */
 	__u32 flags;
 	/* source picture description */
 	__u32 bo_handle;
 	/* stride values and offsets are in bytes, buffer relative */
 	__u16 stride_Y; /* stride for packed formats */
 	__u16 stride_UV;
 	__u32 offset_Y; /* offset for packet formats */
 	__u32 offset_U;
 	__u32 offset_V;
 	/* in pixels */
 	__u16 src_width;
 	__u16 src_height;
 	/* to compensate the scaling factors for partially covered surfaces */
 	__u16 src_scan_width;
 	__u16 src_scan_height;
 	/* output crtc description */
 	__u32 crtc_id;
 	__u16 dst_x;
 	__u16 dst_y;
 	__u16 dst_width;
 	__u16 dst_height;
 };
 /* flags */
 #define I915_OVERLAY_UPDATE_ATTRS	(1<<0)
 #define I915_OVERLAY_UPDATE_GAMMA	(1<<1)
 struct drm_intel_overlay_attrs {
 	__u32 flags;
 	__u32 color_key;
 	__s32 brightness;
 	__u32 contrast;
 	__u32 saturation;
 	__u32 gamma0;
 	__u32 gamma1;
 	__u32 gamma2;
 	__u32 gamma3;
 	__u32 gamma4;
 	__u32 gamma5;
 };
 /*
  * Intel sprite handling
  *
  * Color keying works with a min/mask/max tuple.  Both source and destination
  * color keying is allowed.
  *
  * Source keying:
  * Sprite pixels within the min & max values, masked against the color channels
  * specified in the mask field, will be transparent.  All other pixels will
  * be displayed on top of the primary plane.  For RGB surfaces, only the min
  * and mask fields will be used; ranged compares are not allowed.
  *
  * Destination keying:
  * Primary plane pixels that match the min value, masked against the color
  * channels specified in the mask field, will be replaced by corresponding
  * pixels from the sprite plane.
  *
  * Note that source & destination keying are exclusive; only one can be
  * active on a given plane.
  */
 #define I915_SET_COLORKEY_NONE		(1<<0) /* disable color key matching */
 #define I915_SET_COLORKEY_DESTINATION	(1<<1)
 #define I915_SET_COLORKEY_SOURCE	(1<<2)
 struct drm_intel_sprite_colorkey {
 	__u32 plane_id;
 	__u32 min_value;
 	__u32 channel_mask;
 	__u32 max_value;
 	__u32 flags;
 };
 #endif				/* _I915_DRM_H_ */