Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 7a1619b97e978bb9c05fa4bbe64171068bd5bf85

Authored by Michel Dänzer 2011-11-11 01:57:26 +0800

Committed by Dave Airlie 2011-11-11 19:02:10 +0800

Exists in master and in 6 other branches

drm/radeon: Make sure CS mutex is held across GPU reset.

This was only the case if the GPU reset was triggered from the CS ioctl,
otherwise other processes could happily enter the CS ioctl and wreak havoc
during the GPU reset.

This is a little complicated because the GPU reset can be triggered from the
CS ioctl, in which case we're already holding the mutex, or from other call
paths, in which case we need to lock the mutex. AFAICT the mutex API doesn't
allow recursive locking or finding out the mutex owner, so we need to handle
this with helper functions which allow recursive locking from the same
process.

Signed-off-by: Michel Dänzer <michel.daenzer@amd.com>
Reviewed-by: Jerome Glisse <jglisse@redhat.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

Showing 3 changed files with 62 additions and 12 deletions Inline Diff

drivers/gpu/drm/radeon/radeon.h
drivers/gpu/drm/radeon/radeon_cs.c
drivers/gpu/drm/radeon/radeon_device.c

drivers/gpu/drm/radeon/radeon.h

Diff comments View file @ 7a1619b

 /*
  * Copyright 2008 Advanced Micro Devices, Inc.
  * Copyright 2008 Red Hat Inc.
  * Copyright 2009 Jerome Glisse.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in
  * all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  * Authors: Dave Airlie
  *          Alex Deucher
  *          Jerome Glisse
  */
 #ifndef __RADEON_H__
 #define __RADEON_H__
 /* TODO: Here are things that needs to be done :
  *	- surface allocator & initializer : (bit like scratch reg) should
  *	  initialize HDP_ stuff on RS600, R600, R700 hw, well anythings
  *	  related to surface
  *	- WB : write back stuff (do it bit like scratch reg things)
  *	- Vblank : look at Jesse's rework and what we should do
  *	- r600/r700: gart & cp
  *	- cs : clean cs ioctl use bitmap & things like that.
  *	- power management stuff
  *	- Barrier in gart code
  *	- Unmappabled vram ?
  *	- TESTING, TESTING, TESTING
  */
 /* Initialization path:
  *  We expect that acceleration initialization might fail for various
  *  reasons even thought we work hard to make it works on most
  *  configurations. In order to still have a working userspace in such
  *  situation the init path must succeed up to the memory controller
  *  initialization point. Failure before this point are considered as
  *  fatal error. Here is the init callchain :
  *      radeon_device_init  perform common structure, mutex initialization
  *      asic_init           setup the GPU memory layout and perform all
  *                          one time initialization (failure in this
  *                          function are considered fatal)
  *      asic_startup        setup the GPU acceleration, in order to
  *                          follow guideline the first thing this
  *                          function should do is setting the GPU
  *                          memory controller (only MC setup failure
  *                          are considered as fatal)
  */
 #include <linux/atomic.h>
 #include <linux/wait.h>
 #include <linux/list.h>
 #include <linux/kref.h>
 #include <ttm/ttm_bo_api.h>
 #include <ttm/ttm_bo_driver.h>
 #include <ttm/ttm_placement.h>
 #include <ttm/ttm_module.h>
 #include <ttm/ttm_execbuf_util.h>
 #include "radeon_family.h"
 #include "radeon_mode.h"
 #include "radeon_reg.h"
 /*
  * Modules parameters.
  */
 extern int radeon_no_wb;
 extern int radeon_modeset;
 extern int radeon_dynclks;
 extern int radeon_r4xx_atom;
 extern int radeon_agpmode;
 extern int radeon_vram_limit;
 extern int radeon_gart_size;
 extern int radeon_benchmarking;
 extern int radeon_testing;
 extern int radeon_connector_table;
 extern int radeon_tv;
 extern int radeon_audio;
 extern int radeon_disp_priority;
 extern int radeon_hw_i2c;
 extern int radeon_pcie_gen2;
 extern int radeon_msi;
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
  * symbol;
  */
 #define RADEON_MAX_USEC_TIMEOUT		100000	/* 100 ms */
 #define RADEON_FENCE_JIFFIES_TIMEOUT	(HZ / 2)
 /* RADEON_IB_POOL_SIZE must be a power of 2 */
 #define RADEON_IB_POOL_SIZE		16
 #define RADEON_DEBUGFS_MAX_COMPONENTS	32
 #define RADEONFB_CONN_LIMIT		4
 #define RADEON_BIOS_NUM_SCRATCH		8
 /*
  * Errata workarounds.
  */
 enum radeon_pll_errata {
 	CHIP_ERRATA_R300_CG             = 0x00000001,
 	CHIP_ERRATA_PLL_DUMMYREADS      = 0x00000002,
 	CHIP_ERRATA_PLL_DELAY           = 0x00000004
 };
 struct radeon_device;
 /*
  * BIOS.
  */
 #define ATRM_BIOS_PAGE 4096
 #if defined(CONFIG_VGA_SWITCHEROO)
 bool radeon_atrm_supported(struct pci_dev *pdev);
 int radeon_atrm_get_bios_chunk(uint8_t *bios, int offset, int len);
 #else
 static inline bool radeon_atrm_supported(struct pci_dev *pdev)
 {
 	return false;
 }
 static inline int radeon_atrm_get_bios_chunk(uint8_t *bios, int offset, int len){
 	return -EINVAL;
 }
 #endif
 bool radeon_get_bios(struct radeon_device *rdev);
 /*
  * Dummy page
  */
 struct radeon_dummy_page {
 	struct page	*page;
 	dma_addr_t	addr;
 };
 int radeon_dummy_page_init(struct radeon_device *rdev);
 void radeon_dummy_page_fini(struct radeon_device *rdev);
 /*
  * Clocks
  */
 struct radeon_clock {
 	struct radeon_pll p1pll;
 	struct radeon_pll p2pll;
 	struct radeon_pll dcpll;
 	struct radeon_pll spll;
 	struct radeon_pll mpll;
 	/* 10 Khz units */
 	uint32_t default_mclk;
 	uint32_t default_sclk;
 	uint32_t default_dispclk;
 	uint32_t dp_extclk;
 	uint32_t max_pixel_clock;
 };
 /*
  * Power management
  */
 int radeon_pm_init(struct radeon_device *rdev);
 void radeon_pm_fini(struct radeon_device *rdev);
 void radeon_pm_compute_clocks(struct radeon_device *rdev);
 void radeon_pm_suspend(struct radeon_device *rdev);
 void radeon_pm_resume(struct radeon_device *rdev);
 void radeon_combios_get_power_modes(struct radeon_device *rdev);
 void radeon_atombios_get_power_modes(struct radeon_device *rdev);
 void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
 int radeon_atom_get_max_vddc(struct radeon_device *rdev, u16 *voltage);
 void rs690_pm_info(struct radeon_device *rdev);
 extern int rv6xx_get_temp(struct radeon_device *rdev);
 extern int rv770_get_temp(struct radeon_device *rdev);
 extern int evergreen_get_temp(struct radeon_device *rdev);
 extern int sumo_get_temp(struct radeon_device *rdev);
 /*
  * Fences.
  */
 struct radeon_fence_driver {
 	uint32_t			scratch_reg;
 	atomic_t			seq;
 	uint32_t			last_seq;
 	unsigned long			last_jiffies;
 	unsigned long			last_timeout;
 	wait_queue_head_t		queue;
 	rwlock_t			lock;
 	struct list_head		created;
 	struct list_head		emited;
 	struct list_head		signaled;
 	bool				initialized;
 };
 struct radeon_fence {
 	struct radeon_device		*rdev;
 	struct kref			kref;
 	struct list_head		list;
 	/* protected by radeon_fence.lock */
 	uint32_t			seq;
 	bool				emited;
 	bool				signaled;
 };
 int radeon_fence_driver_init(struct radeon_device *rdev);
 void radeon_fence_driver_fini(struct radeon_device *rdev);
 int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence);
 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence);
 void radeon_fence_process(struct radeon_device *rdev);
 bool radeon_fence_signaled(struct radeon_fence *fence);
 int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
 int radeon_fence_wait_next(struct radeon_device *rdev);
 int radeon_fence_wait_last(struct radeon_device *rdev);
 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence);
 void radeon_fence_unref(struct radeon_fence **fence);
 /*
  * Tiling registers
  */
 struct radeon_surface_reg {
 	struct radeon_bo *bo;
 };
 #define RADEON_GEM_MAX_SURFACES 8
 /*
  * TTM.
  */
 struct radeon_mman {
 	struct ttm_bo_global_ref        bo_global_ref;
 	struct drm_global_reference	mem_global_ref;
 	struct ttm_bo_device		bdev;
 	bool				mem_global_referenced;
 	bool				initialized;
 };
 struct radeon_bo {
 	/* Protected by gem.mutex */
 	struct list_head		list;
 	/* Protected by tbo.reserved */
 	u32				placements[3];
 	struct ttm_placement		placement;
 	struct ttm_buffer_object	tbo;
 	struct ttm_bo_kmap_obj		kmap;
 	unsigned			pin_count;
 	void				*kptr;
 	u32				tiling_flags;
 	u32				pitch;
 	int				surface_reg;
 	/* Constant after initialization */
 	struct radeon_device		*rdev;
 	struct drm_gem_object		gem_base;
 };
 #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
 struct radeon_bo_list {
 	struct ttm_validate_buffer tv;
 	struct radeon_bo	*bo;
 	uint64_t		gpu_offset;
 	unsigned		rdomain;
 	unsigned		wdomain;
 	u32			tiling_flags;
 };
 /*
  * GEM objects.
  */
 struct radeon_gem {
 	struct mutex		mutex;
 	struct list_head	objects;
 };
 int radeon_gem_init(struct radeon_device *rdev);
 void radeon_gem_fini(struct radeon_device *rdev);
 int radeon_gem_object_create(struct radeon_device *rdev, int size,
 				int alignment, int initial_domain,
 				bool discardable, bool kernel,
 				struct drm_gem_object **obj);
 int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
 			  uint64_t *gpu_addr);
 void radeon_gem_object_unpin(struct drm_gem_object *obj);
 int radeon_mode_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *dev,
 			    struct drm_mode_create_dumb *args);
 int radeon_mode_dumb_mmap(struct drm_file *filp,
 			  struct drm_device *dev,
 			  uint32_t handle, uint64_t *offset_p);
 int radeon_mode_dumb_destroy(struct drm_file *file_priv,
 			     struct drm_device *dev,
 			     uint32_t handle);
 /*
  * GART structures, functions & helpers
  */
 struct radeon_mc;
 #define RADEON_GPU_PAGE_SIZE 4096
 #define RADEON_GPU_PAGE_MASK (RADEON_GPU_PAGE_SIZE - 1)
 #define RADEON_GPU_PAGE_SHIFT 12
 struct radeon_gart {
 	dma_addr_t			table_addr;
 	struct radeon_bo		*robj;
 	void				*ptr;
 	unsigned			num_gpu_pages;
 	unsigned			num_cpu_pages;
 	unsigned			table_size;
 	struct page			**pages;
 	dma_addr_t			*pages_addr;
 	bool				*ttm_alloced;
 	bool				ready;
 };
 int radeon_gart_table_ram_alloc(struct radeon_device *rdev);
 void radeon_gart_table_ram_free(struct radeon_device *rdev);
 int radeon_gart_table_vram_alloc(struct radeon_device *rdev);
 void radeon_gart_table_vram_free(struct radeon_device *rdev);
 int radeon_gart_table_vram_pin(struct radeon_device *rdev);
 void radeon_gart_table_vram_unpin(struct radeon_device *rdev);
 int radeon_gart_init(struct radeon_device *rdev);
 void radeon_gart_fini(struct radeon_device *rdev);
 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
 			int pages);
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
 		     int pages, struct page **pagelist,
 		     dma_addr_t *dma_addr);
 void radeon_gart_restore(struct radeon_device *rdev);
 /*
  * GPU MC structures, functions & helpers
  */
 struct radeon_mc {
 	resource_size_t		aper_size;
 	resource_size_t		aper_base;
 	resource_size_t		agp_base;
 	/* for some chips with <= 32MB we need to lie
 	 * about vram size near mc fb location */
 	u64			mc_vram_size;
 	u64			visible_vram_size;
 	u64			gtt_size;
 	u64			gtt_start;
 	u64			gtt_end;
 	u64			vram_start;
 	u64			vram_end;
 	unsigned		vram_width;
 	u64			real_vram_size;
 	int			vram_mtrr;
 	bool			vram_is_ddr;
 	bool			igp_sideport_enabled;
 	u64                     gtt_base_align;
 };
 bool radeon_combios_sideport_present(struct radeon_device *rdev);
 bool radeon_atombios_sideport_present(struct radeon_device *rdev);
 /*
  * GPU scratch registers structures, functions & helpers
  */
 struct radeon_scratch {
 	unsigned		num_reg;
 	uint32_t                reg_base;
 	bool			free[32];
 	uint32_t		reg[32];
 };
 int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg);
 void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
 /*
  * IRQS.
  */
 struct radeon_unpin_work {
 	struct work_struct work;
 	struct radeon_device *rdev;
 	int crtc_id;
 	struct radeon_fence *fence;
 	struct drm_pending_vblank_event *event;
 	struct radeon_bo *old_rbo;
 	u64 new_crtc_base;
 };
 struct r500_irq_stat_regs {
 	u32 disp_int;
 };
 struct r600_irq_stat_regs {
 	u32 disp_int;
 	u32 disp_int_cont;
 	u32 disp_int_cont2;
 	u32 d1grph_int;
 	u32 d2grph_int;
 };
 struct evergreen_irq_stat_regs {
 	u32 disp_int;
 	u32 disp_int_cont;
 	u32 disp_int_cont2;
 	u32 disp_int_cont3;
 	u32 disp_int_cont4;
 	u32 disp_int_cont5;
 	u32 d1grph_int;
 	u32 d2grph_int;
 	u32 d3grph_int;
 	u32 d4grph_int;
 	u32 d5grph_int;
 	u32 d6grph_int;
 };
 union radeon_irq_stat_regs {
 	struct r500_irq_stat_regs r500;
 	struct r600_irq_stat_regs r600;
 	struct evergreen_irq_stat_regs evergreen;
 };
 #define RADEON_MAX_HPD_PINS 6
 #define RADEON_MAX_CRTCS 6
 #define RADEON_MAX_HDMI_BLOCKS 2
 struct radeon_irq {
 	bool		installed;
 	bool		sw_int;
 	bool		crtc_vblank_int[RADEON_MAX_CRTCS];
 	bool		pflip[RADEON_MAX_CRTCS];
 	wait_queue_head_t	vblank_queue;
 	bool            hpd[RADEON_MAX_HPD_PINS];
 	bool            gui_idle;
 	bool            gui_idle_acked;
 	wait_queue_head_t	idle_queue;
 	bool		hdmi[RADEON_MAX_HDMI_BLOCKS];
 	spinlock_t sw_lock;
 	int sw_refcount;
 	union radeon_irq_stat_regs stat_regs;
 	spinlock_t pflip_lock[RADEON_MAX_CRTCS];
 	int pflip_refcount[RADEON_MAX_CRTCS];
 };
 int radeon_irq_kms_init(struct radeon_device *rdev);
 void radeon_irq_kms_fini(struct radeon_device *rdev);
 void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev);
 void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev);
 void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
 void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
 /*
  * CP & ring.
  */
 struct radeon_ib {
 	struct list_head	list;
 	unsigned		idx;
 	uint64_t		gpu_addr;
 	struct radeon_fence	*fence;
 	uint32_t		*ptr;
 	uint32_t		length_dw;
 	bool			free;
 };
 /*
  * locking -
  * mutex protects scheduled_ibs, ready, alloc_bm
  */
 struct radeon_ib_pool {
 	struct mutex		mutex;
 	struct radeon_bo	*robj;
 	struct list_head	bogus_ib;
 	struct radeon_ib	ibs[RADEON_IB_POOL_SIZE];
 	bool			ready;
 	unsigned		head_id;
 };
 struct radeon_cp {
 	struct radeon_bo	*ring_obj;
 	volatile uint32_t	*ring;
 	unsigned		rptr;
 	unsigned		wptr;
 	unsigned		wptr_old;
 	unsigned		ring_size;
 	unsigned		ring_free_dw;
 	int			count_dw;
 	uint64_t		gpu_addr;
 	uint32_t		align_mask;
 	uint32_t		ptr_mask;
 	struct mutex		mutex;
 	bool			ready;
 };
 /*
  * R6xx+ IH ring
  */
 struct r600_ih {
 	struct radeon_bo	*ring_obj;
 	volatile uint32_t	*ring;
 	unsigned		rptr;
 	unsigned		wptr;
 	unsigned		wptr_old;
 	unsigned		ring_size;
 	uint64_t		gpu_addr;
 	uint32_t		ptr_mask;
 	spinlock_t              lock;
 	bool                    enabled;
 };
 struct r600_blit_cp_primitives {
 	void (*set_render_target)(struct radeon_device *rdev, int format,
 				  int w, int h, u64 gpu_addr);
 	void (*cp_set_surface_sync)(struct radeon_device *rdev,
 				    u32 sync_type, u32 size,
 				    u64 mc_addr);
 	void (*set_shaders)(struct radeon_device *rdev);
 	void (*set_vtx_resource)(struct radeon_device *rdev, u64 gpu_addr);
 	void (*set_tex_resource)(struct radeon_device *rdev,
 				 int format, int w, int h, int pitch,
 				 u64 gpu_addr, u32 size);
 	void (*set_scissors)(struct radeon_device *rdev, int x1, int y1,
 			     int x2, int y2);
 	void (*draw_auto)(struct radeon_device *rdev);
 	void (*set_default_state)(struct radeon_device *rdev);
 };
 struct r600_blit {
 	struct mutex		mutex;
 	struct radeon_bo	*shader_obj;
 	struct r600_blit_cp_primitives primitives;
 	int max_dim;
 	int ring_size_common;
 	int ring_size_per_loop;
 	u64 shader_gpu_addr;
 	u32 vs_offset, ps_offset;
 	u32 state_offset;
 	u32 state_len;
 	u32 vb_used, vb_total;
 	struct radeon_ib *vb_ib;
 };
 void r600_blit_suspend(struct radeon_device *rdev);
 int radeon_ib_get(struct radeon_device *rdev, struct radeon_ib **ib);
 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib);
 int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib);
 int radeon_ib_pool_init(struct radeon_device *rdev);
 void radeon_ib_pool_fini(struct radeon_device *rdev);
 int radeon_ib_test(struct radeon_device *rdev);
 extern void radeon_ib_bogus_add(struct radeon_device *rdev, struct radeon_ib *ib);
 /* Ring access between begin & end cannot sleep */
 void radeon_ring_free_size(struct radeon_device *rdev);
 int radeon_ring_alloc(struct radeon_device *rdev, unsigned ndw);
 int radeon_ring_lock(struct radeon_device *rdev, unsigned ndw);
 void radeon_ring_commit(struct radeon_device *rdev);
 void radeon_ring_unlock_commit(struct radeon_device *rdev);
 void radeon_ring_unlock_undo(struct radeon_device *rdev);
 int radeon_ring_test(struct radeon_device *rdev);
 int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size);
 void radeon_ring_fini(struct radeon_device *rdev);
 /*
  * CS.
  */
 struct radeon_cs_reloc {
 	struct drm_gem_object		*gobj;
 	struct radeon_bo		*robj;
 	struct radeon_bo_list		lobj;
 	uint32_t			handle;
 	uint32_t			flags;
 };
 struct radeon_cs_chunk {
 	uint32_t		chunk_id;
 	uint32_t		length_dw;
 	int kpage_idx[2];
 	uint32_t                *kpage[2];
 	uint32_t		*kdata;
 	void __user *user_ptr;
 	int last_copied_page;
 	int last_page_index;
 };
 struct radeon_cs_parser {
 	struct device		*dev;
 	struct radeon_device	*rdev;
 	struct drm_file		*filp;
 	/* chunks */
 	unsigned		nchunks;
 	struct radeon_cs_chunk	*chunks;
 	uint64_t		*chunks_array;
 	/* IB */
 	unsigned		idx;
 	/* relocations */
 	unsigned		nrelocs;
 	struct radeon_cs_reloc	*relocs;
 	struct radeon_cs_reloc	**relocs_ptr;
 	struct list_head	validated;
 	/* indices of various chunks */
 	int			chunk_ib_idx;
 	int			chunk_relocs_idx;
 	struct radeon_ib	*ib;
 	void			*track;
 	unsigned		family;
 	int parser_error;
 };
 extern int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx);
 extern int radeon_cs_finish_pages(struct radeon_cs_parser *p);
 extern u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx);
 struct radeon_cs_packet {
 	unsigned	idx;
 	unsigned	type;
 	unsigned	reg;
 	unsigned	opcode;
 	int		count;
 	unsigned	one_reg_wr;
 };
 typedef int (*radeon_packet0_check_t)(struct radeon_cs_parser *p,
 				      struct radeon_cs_packet *pkt,
 				      unsigned idx, unsigned reg);
 typedef int (*radeon_packet3_check_t)(struct radeon_cs_parser *p,
 				      struct radeon_cs_packet *pkt);
 /*
  * AGP
  */
 int radeon_agp_init(struct radeon_device *rdev);
 void radeon_agp_resume(struct radeon_device *rdev);
 void radeon_agp_suspend(struct radeon_device *rdev);
 void radeon_agp_fini(struct radeon_device *rdev);
 /*
  * Writeback
  */
 struct radeon_wb {
 	struct radeon_bo	*wb_obj;
 	volatile uint32_t	*wb;
 	uint64_t		gpu_addr;
 	bool                    enabled;
 	bool                    use_event;
 };
 #define RADEON_WB_SCRATCH_OFFSET 0
 #define RADEON_WB_CP_RPTR_OFFSET 1024
 #define RADEON_WB_CP1_RPTR_OFFSET 1280
 #define RADEON_WB_CP2_RPTR_OFFSET 1536
 #define R600_WB_IH_WPTR_OFFSET   2048
 #define R600_WB_EVENT_OFFSET     3072
 /**
  * struct radeon_pm - power management datas
  * @max_bandwidth:      maximum bandwidth the gpu has (MByte/s)
  * @igp_sideport_mclk:  sideport memory clock Mhz (rs690,rs740,rs780,rs880)
  * @igp_system_mclk:    system clock Mhz (rs690,rs740,rs780,rs880)
  * @igp_ht_link_clk:    ht link clock Mhz (rs690,rs740,rs780,rs880)
  * @igp_ht_link_width:  ht link width in bits (rs690,rs740,rs780,rs880)
  * @k8_bandwidth:       k8 bandwidth the gpu has (MByte/s) (IGP)
  * @sideport_bandwidth: sideport bandwidth the gpu has (MByte/s) (IGP)
  * @ht_bandwidth:       ht bandwidth the gpu has (MByte/s) (IGP)
  * @core_bandwidth:     core GPU bandwidth the gpu has (MByte/s) (IGP)
  * @sclk:          	GPU clock Mhz (core bandwidth depends of this clock)
  * @needed_bandwidth:   current bandwidth needs
  *
  * It keeps track of various data needed to take powermanagement decision.
  * Bandwidth need is used to determine minimun clock of the GPU and memory.
  * Equation between gpu/memory clock and available bandwidth is hw dependent
  * (type of memory, bus size, efficiency, ...)
  */
 enum radeon_pm_method {
 	PM_METHOD_PROFILE,
 	PM_METHOD_DYNPM,
 };
 enum radeon_dynpm_state {
 	DYNPM_STATE_DISABLED,
 	DYNPM_STATE_MINIMUM,
 	DYNPM_STATE_PAUSED,
 	DYNPM_STATE_ACTIVE,
 	DYNPM_STATE_SUSPENDED,
 };
 enum radeon_dynpm_action {
 	DYNPM_ACTION_NONE,
 	DYNPM_ACTION_MINIMUM,
 	DYNPM_ACTION_DOWNCLOCK,
 	DYNPM_ACTION_UPCLOCK,
 	DYNPM_ACTION_DEFAULT
 };
 enum radeon_voltage_type {
 	VOLTAGE_NONE = 0,
 	VOLTAGE_GPIO,
 	VOLTAGE_VDDC,
 	VOLTAGE_SW
 };
 enum radeon_pm_state_type {
 	POWER_STATE_TYPE_DEFAULT,
 	POWER_STATE_TYPE_POWERSAVE,
 	POWER_STATE_TYPE_BATTERY,
 	POWER_STATE_TYPE_BALANCED,
 	POWER_STATE_TYPE_PERFORMANCE,
 };
 enum radeon_pm_profile_type {
 	PM_PROFILE_DEFAULT,
 	PM_PROFILE_AUTO,
 	PM_PROFILE_LOW,
 	PM_PROFILE_MID,
 	PM_PROFILE_HIGH,
 };
 #define PM_PROFILE_DEFAULT_IDX 0
 #define PM_PROFILE_LOW_SH_IDX  1
 #define PM_PROFILE_MID_SH_IDX  2
 #define PM_PROFILE_HIGH_SH_IDX 3
 #define PM_PROFILE_LOW_MH_IDX  4
 #define PM_PROFILE_MID_MH_IDX  5
 #define PM_PROFILE_HIGH_MH_IDX 6
 #define PM_PROFILE_MAX         7
 struct radeon_pm_profile {
 	int dpms_off_ps_idx;
 	int dpms_on_ps_idx;
 	int dpms_off_cm_idx;
 	int dpms_on_cm_idx;
 };
 enum radeon_int_thermal_type {
 	THERMAL_TYPE_NONE,
 	THERMAL_TYPE_RV6XX,
 	THERMAL_TYPE_RV770,
 	THERMAL_TYPE_EVERGREEN,
 	THERMAL_TYPE_SUMO,
 	THERMAL_TYPE_NI,
 };
 struct radeon_voltage {
 	enum radeon_voltage_type type;
 	/* gpio voltage */
 	struct radeon_gpio_rec gpio;
 	u32 delay; /* delay in usec from voltage drop to sclk change */
 	bool active_high; /* voltage drop is active when bit is high */
 	/* VDDC voltage */
 	u8 vddc_id; /* index into vddc voltage table */
 	u8 vddci_id; /* index into vddci voltage table */
 	bool vddci_enabled;
 	/* r6xx+ sw */
 	u16 voltage;
 	/* evergreen+ vddci */
 	u16 vddci;
 };
 /* clock mode flags */
 #define RADEON_PM_MODE_NO_DISPLAY          (1 << 0)
 struct radeon_pm_clock_info {
 	/* memory clock */
 	u32 mclk;
 	/* engine clock */
 	u32 sclk;
 	/* voltage info */
 	struct radeon_voltage voltage;
 	/* standardized clock flags */
 	u32 flags;
 };
 /* state flags */
 #define RADEON_PM_STATE_SINGLE_DISPLAY_ONLY (1 << 0)
 struct radeon_power_state {
 	enum radeon_pm_state_type type;
 	/* XXX: use a define for num clock modes */
 	struct radeon_pm_clock_info clock_info[8];
 	/* number of valid clock modes in this power state */
 	int num_clock_modes;
 	struct radeon_pm_clock_info *default_clock_mode;
 	/* standardized state flags */
 	u32 flags;
 	u32 misc; /* vbios specific flags */
 	u32 misc2; /* vbios specific flags */
 	int pcie_lanes; /* pcie lanes */
 };
 /*
  * Some modes are overclocked by very low value, accept them
  */
 #define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */
 struct radeon_pm {
 	struct mutex		mutex;
 	u32			active_crtcs;
 	int			active_crtc_count;
 	int			req_vblank;
 	bool			vblank_sync;
 	bool			gui_idle;
 	fixed20_12		max_bandwidth;
 	fixed20_12		igp_sideport_mclk;
 	fixed20_12		igp_system_mclk;
 	fixed20_12		igp_ht_link_clk;
 	fixed20_12		igp_ht_link_width;
 	fixed20_12		k8_bandwidth;
 	fixed20_12		sideport_bandwidth;
 	fixed20_12		ht_bandwidth;
 	fixed20_12		core_bandwidth;
 	fixed20_12		sclk;
 	fixed20_12		mclk;
 	fixed20_12		needed_bandwidth;
 	struct radeon_power_state *power_state;
 	/* number of valid power states */
 	int                     num_power_states;
 	int                     current_power_state_index;
 	int                     current_clock_mode_index;
 	int                     requested_power_state_index;
 	int                     requested_clock_mode_index;
 	int                     default_power_state_index;
 	u32                     current_sclk;
 	u32                     current_mclk;
 	u16                     current_vddc;
 	u16                     current_vddci;
 	u32                     default_sclk;
 	u32                     default_mclk;
 	u16                     default_vddc;
 	u16                     default_vddci;
 	struct radeon_i2c_chan *i2c_bus;
 	/* selected pm method */
 	enum radeon_pm_method     pm_method;
 	/* dynpm power management */
 	struct delayed_work	dynpm_idle_work;
 	enum radeon_dynpm_state	dynpm_state;
 	enum radeon_dynpm_action	dynpm_planned_action;
 	unsigned long		dynpm_action_timeout;
 	bool                    dynpm_can_upclock;
 	bool                    dynpm_can_downclock;
 	/* profile-based power management */
 	enum radeon_pm_profile_type profile;
 	int                     profile_index;
 	struct radeon_pm_profile profiles[PM_PROFILE_MAX];
 	/* internal thermal controller on rv6xx+ */
 	enum radeon_int_thermal_type int_thermal_type;
 	struct device	        *int_hwmon_dev;
 };
 /*
  * Benchmarking
  */
 void radeon_benchmark(struct radeon_device *rdev, int test_number);
 /*
  * Testing
  */
 void radeon_test_moves(struct radeon_device *rdev);
 /*
  * Debugfs
  */
 int radeon_debugfs_add_files(struct radeon_device *rdev,
 			     struct drm_info_list *files,
 			     unsigned nfiles);
 int radeon_debugfs_fence_init(struct radeon_device *rdev);
 /*
  * ASIC specific functions.
  */
 struct radeon_asic {
 	int (*init)(struct radeon_device *rdev);
 	void (*fini)(struct radeon_device *rdev);
 	int (*resume)(struct radeon_device *rdev);
 	int (*suspend)(struct radeon_device *rdev);
 	void (*vga_set_state)(struct radeon_device *rdev, bool state);
 	bool (*gpu_is_lockup)(struct radeon_device *rdev);
 	int (*asic_reset)(struct radeon_device *rdev);
 	void (*gart_tlb_flush)(struct radeon_device *rdev);
 	int (*gart_set_page)(struct radeon_device *rdev, int i, uint64_t addr);
 	int (*cp_init)(struct radeon_device *rdev, unsigned ring_size);
 	void (*cp_fini)(struct radeon_device *rdev);
 	void (*cp_disable)(struct radeon_device *rdev);
 	void (*cp_commit)(struct radeon_device *rdev);
 	void (*ring_start)(struct radeon_device *rdev);
 	int (*ring_test)(struct radeon_device *rdev);
 	void (*ring_ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib);
 	int (*irq_set)(struct radeon_device *rdev);
 	int (*irq_process)(struct radeon_device *rdev);
 	u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
 	void (*fence_ring_emit)(struct radeon_device *rdev, struct radeon_fence *fence);
 	int (*cs_parse)(struct radeon_cs_parser *p);
 	int (*copy_blit)(struct radeon_device *rdev,
 			 uint64_t src_offset,
 			 uint64_t dst_offset,
 			 unsigned num_gpu_pages,
 			 struct radeon_fence *fence);
 	int (*copy_dma)(struct radeon_device *rdev,
 			uint64_t src_offset,
 			uint64_t dst_offset,
 			unsigned num_gpu_pages,
 			struct radeon_fence *fence);
 	int (*copy)(struct radeon_device *rdev,
 		    uint64_t src_offset,
 		    uint64_t dst_offset,
 		    unsigned num_gpu_pages,
 		    struct radeon_fence *fence);
 	uint32_t (*get_engine_clock)(struct radeon_device *rdev);
 	void (*set_engine_clock)(struct radeon_device *rdev, uint32_t eng_clock);
 	uint32_t (*get_memory_clock)(struct radeon_device *rdev);
 	void (*set_memory_clock)(struct radeon_device *rdev, uint32_t mem_clock);
 	int (*get_pcie_lanes)(struct radeon_device *rdev);
 	void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes);
 	void (*set_clock_gating)(struct radeon_device *rdev, int enable);
 	int (*set_surface_reg)(struct radeon_device *rdev, int reg,
 			       uint32_t tiling_flags, uint32_t pitch,
 			       uint32_t offset, uint32_t obj_size);
 	void (*clear_surface_reg)(struct radeon_device *rdev, int reg);
 	void (*bandwidth_update)(struct radeon_device *rdev);
 	void (*hpd_init)(struct radeon_device *rdev);
 	void (*hpd_fini)(struct radeon_device *rdev);
 	bool (*hpd_sense)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
 	void (*hpd_set_polarity)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
 	/* ioctl hw specific callback. Some hw might want to perform special
 	 * operation on specific ioctl. For instance on wait idle some hw
 	 * might want to perform and HDP flush through MMIO as it seems that
 	 * some R6XX/R7XX hw doesn't take HDP flush into account if programmed
 	 * through ring.
 	 */
 	void (*ioctl_wait_idle)(struct radeon_device *rdev, struct radeon_bo *bo);
 	bool (*gui_idle)(struct radeon_device *rdev);
 	/* power management */
 	void (*pm_misc)(struct radeon_device *rdev);
 	void (*pm_prepare)(struct radeon_device *rdev);
 	void (*pm_finish)(struct radeon_device *rdev);
 	void (*pm_init_profile)(struct radeon_device *rdev);
 	void (*pm_get_dynpm_state)(struct radeon_device *rdev);
 	/* pageflipping */
 	void (*pre_page_flip)(struct radeon_device *rdev, int crtc);
 	u32 (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base);
 	void (*post_page_flip)(struct radeon_device *rdev, int crtc);
 };
 /*
  * Asic structures
  */
 struct r100_gpu_lockup {
 	unsigned long	last_jiffies;
 	u32		last_cp_rptr;
 };
 struct r100_asic {
 	const unsigned		*reg_safe_bm;
 	unsigned		reg_safe_bm_size;
 	u32			hdp_cntl;
 	struct r100_gpu_lockup	lockup;
 };
 struct r300_asic {
 	const unsigned		*reg_safe_bm;
 	unsigned		reg_safe_bm_size;
 	u32			resync_scratch;
 	u32			hdp_cntl;
 	struct r100_gpu_lockup	lockup;
 };
 struct r600_asic {
 	unsigned		max_pipes;
 	unsigned		max_tile_pipes;
 	unsigned		max_simds;
 	unsigned		max_backends;
 	unsigned		max_gprs;
 	unsigned		max_threads;
 	unsigned		max_stack_entries;
 	unsigned		max_hw_contexts;
 	unsigned		max_gs_threads;
 	unsigned		sx_max_export_size;
 	unsigned		sx_max_export_pos_size;
 	unsigned		sx_max_export_smx_size;
 	unsigned		sq_num_cf_insts;
 	unsigned		tiling_nbanks;
 	unsigned		tiling_npipes;
 	unsigned		tiling_group_size;
 	unsigned		tile_config;
 	unsigned		backend_map;
 	struct r100_gpu_lockup	lockup;
 };
 struct rv770_asic {
 	unsigned		max_pipes;
 	unsigned		max_tile_pipes;
 	unsigned		max_simds;
 	unsigned		max_backends;
 	unsigned		max_gprs;
 	unsigned		max_threads;
 	unsigned		max_stack_entries;
 	unsigned		max_hw_contexts;
 	unsigned		max_gs_threads;
 	unsigned		sx_max_export_size;
 	unsigned		sx_max_export_pos_size;
 	unsigned		sx_max_export_smx_size;
 	unsigned		sq_num_cf_insts;
 	unsigned		sx_num_of_sets;
 	unsigned		sc_prim_fifo_size;
 	unsigned		sc_hiz_tile_fifo_size;
 	unsigned		sc_earlyz_tile_fifo_fize;
 	unsigned		tiling_nbanks;
 	unsigned		tiling_npipes;
 	unsigned		tiling_group_size;
 	unsigned		tile_config;
 	unsigned		backend_map;
 	struct r100_gpu_lockup	lockup;
 };
 struct evergreen_asic {
 	unsigned num_ses;
 	unsigned max_pipes;
 	unsigned max_tile_pipes;
 	unsigned max_simds;
 	unsigned max_backends;
 	unsigned max_gprs;
 	unsigned max_threads;
 	unsigned max_stack_entries;
 	unsigned max_hw_contexts;
 	unsigned max_gs_threads;
 	unsigned sx_max_export_size;
 	unsigned sx_max_export_pos_size;
 	unsigned sx_max_export_smx_size;
 	unsigned sq_num_cf_insts;
 	unsigned sx_num_of_sets;
 	unsigned sc_prim_fifo_size;
 	unsigned sc_hiz_tile_fifo_size;
 	unsigned sc_earlyz_tile_fifo_size;
 	unsigned tiling_nbanks;
 	unsigned tiling_npipes;
 	unsigned tiling_group_size;
 	unsigned tile_config;
 	unsigned backend_map;
 	struct r100_gpu_lockup	lockup;
 };
 struct cayman_asic {
 	unsigned max_shader_engines;
 	unsigned max_pipes_per_simd;
 	unsigned max_tile_pipes;
 	unsigned max_simds_per_se;
 	unsigned max_backends_per_se;
 	unsigned max_texture_channel_caches;
 	unsigned max_gprs;
 	unsigned max_threads;
 	unsigned max_gs_threads;
 	unsigned max_stack_entries;
 	unsigned sx_num_of_sets;
 	unsigned sx_max_export_size;
 	unsigned sx_max_export_pos_size;
 	unsigned sx_max_export_smx_size;
 	unsigned max_hw_contexts;
 	unsigned sq_num_cf_insts;
 	unsigned sc_prim_fifo_size;
 	unsigned sc_hiz_tile_fifo_size;
 	unsigned sc_earlyz_tile_fifo_size;
 	unsigned num_shader_engines;
 	unsigned num_shader_pipes_per_simd;
 	unsigned num_tile_pipes;
 	unsigned num_simds_per_se;
 	unsigned num_backends_per_se;
 	unsigned backend_disable_mask_per_asic;
 	unsigned backend_map;
 	unsigned num_texture_channel_caches;
 	unsigned mem_max_burst_length_bytes;
 	unsigned mem_row_size_in_kb;
 	unsigned shader_engine_tile_size;
 	unsigned num_gpus;
 	unsigned multi_gpu_tile_size;
 	unsigned tile_config;
 	struct r100_gpu_lockup	lockup;
 };
 union radeon_asic_config {
 	struct r300_asic	r300;
 	struct r100_asic	r100;
 	struct r600_asic	r600;
 	struct rv770_asic	rv770;
 	struct evergreen_asic	evergreen;
 	struct cayman_asic	cayman;
 };
 /*
  * asic initizalization from radeon_asic.c
  */
 void radeon_agp_disable(struct radeon_device *rdev);
 int radeon_asic_init(struct radeon_device *rdev);
 /*
  * IOCTL.
  */
 int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *filp);
 int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *filp);
 int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
 int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
 int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp);
 int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *filp);
 int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *filp);
 int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *filp);
 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp);
 int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp);
 /* VRAM scratch page for HDP bug, default vram page */
 struct r600_vram_scratch {
 	struct radeon_bo		*robj;
 	volatile uint32_t		*ptr;
 	u64				gpu_addr;
 };
 /*
+ * Mutex which allows recursive locking from the same process.
+ */
+struct radeon_mutex {
+	struct mutex		mutex;
+	struct task_struct	*owner;
+	int			level;
+};
+static inline void radeon_mutex_init(struct radeon_mutex *mutex)
+{
+	mutex_init(&mutex->mutex);
+	mutex->owner = NULL;
+	mutex->level = 0;
+}
+static inline void radeon_mutex_lock(struct radeon_mutex *mutex)
+{
+	if (mutex_trylock(&mutex->mutex)) {
+		/* The mutex was unlocked before, so it's ours now */
+		mutex->owner = current;
+	} else if (mutex->owner != current) {
+		/* Another process locked the mutex, take it */
+		mutex_lock(&mutex->mutex);
+		mutex->owner = current;
+	}
+	/* Otherwise the mutex was already locked by this process */
+	mutex->level++;
+}
+static inline void radeon_mutex_unlock(struct radeon_mutex *mutex)
+{
+	if (--mutex->level > 0)
+		return;
+	mutex->owner = NULL;
+	mutex_unlock(&mutex->mutex);
+}
+/*
  * Core structure, functions and helpers.
  */
 typedef uint32_t (*radeon_rreg_t)(struct radeon_device*, uint32_t);
 typedef void (*radeon_wreg_t)(struct radeon_device*, uint32_t, uint32_t);
 struct radeon_device {
 	struct device			*dev;
 	struct drm_device		*ddev;
 	struct pci_dev			*pdev;
 	/* ASIC */
 	union radeon_asic_config	config;
 	enum radeon_family		family;
 	unsigned long			flags;
 	int				usec_timeout;
 	enum radeon_pll_errata		pll_errata;
 	int				num_gb_pipes;
 	int				num_z_pipes;
 	int				disp_priority;
 	/* BIOS */
 	uint8_t				*bios;
 	bool				is_atom_bios;
 	uint16_t			bios_header_start;
 	struct radeon_bo		*stollen_vga_memory;
 	/* Register mmio */
 	resource_size_t			rmmio_base;
 	resource_size_t			rmmio_size;
 	void __iomem			*rmmio;
 	radeon_rreg_t			mc_rreg;
 	radeon_wreg_t			mc_wreg;
 	radeon_rreg_t			pll_rreg;
 	radeon_wreg_t			pll_wreg;
 	uint32_t                        pcie_reg_mask;
 	radeon_rreg_t			pciep_rreg;
 	radeon_wreg_t			pciep_wreg;
 	/* io port */
 	void __iomem                    *rio_mem;
 	resource_size_t			rio_mem_size;
 	struct radeon_clock             clock;
 	struct radeon_mc		mc;
 	struct radeon_gart		gart;
 	struct radeon_mode_info		mode_info;
 	struct radeon_scratch		scratch;
 	struct radeon_mman		mman;
 	struct radeon_fence_driver	fence_drv;
 	struct radeon_cp		cp;
 	/* cayman compute rings */
 	struct radeon_cp		cp1;
 	struct radeon_cp		cp2;
 	struct radeon_ib_pool		ib_pool;
 	struct radeon_irq		irq;
 	struct radeon_asic		*asic;
 	struct radeon_gem		gem;
 	struct radeon_pm		pm;
 	uint32_t			bios_scratch[RADEON_BIOS_NUM_SCRATCH];
-	struct mutex			cs_mutex;
+	struct radeon_mutex		cs_mutex;
 	struct radeon_wb		wb;
 	struct radeon_dummy_page	dummy_page;
 	bool				gpu_lockup;
 	bool				shutdown;
 	bool				suspend;
 	bool				need_dma32;
 	bool				accel_working;
 	struct radeon_surface_reg surface_regs[RADEON_GEM_MAX_SURFACES];
 	const struct firmware *me_fw;	/* all family ME firmware */
 	const struct firmware *pfp_fw;	/* r6/700 PFP firmware */
 	const struct firmware *rlc_fw;	/* r6/700 RLC firmware */
 	const struct firmware *mc_fw;	/* NI MC firmware */
 	struct r600_blit r600_blit;
 	struct r600_vram_scratch vram_scratch;
 	int msi_enabled; /* msi enabled */
 	struct r600_ih ih; /* r6/700 interrupt ring */
 	struct work_struct hotplug_work;
 	int num_crtc; /* number of crtcs */
 	struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
 	struct mutex vram_mutex;
 	/* audio stuff */
 	bool			audio_enabled;
 	struct timer_list	audio_timer;
 	int			audio_channels;
 	int			audio_rate;
 	int			audio_bits_per_sample;
 	uint8_t			audio_status_bits;
 	uint8_t			audio_category_code;
 	struct notifier_block acpi_nb;
 	/* only one userspace can use Hyperz features or CMASK at a time */
 	struct drm_file *hyperz_filp;
 	struct drm_file *cmask_filp;
 	/* i2c buses */
 	struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
 };
 int radeon_device_init(struct radeon_device *rdev,
 		       struct drm_device *ddev,
 		       struct pci_dev *pdev,
 		       uint32_t flags);
 void radeon_device_fini(struct radeon_device *rdev);
 int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
 uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg);
 void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 u32 r100_io_rreg(struct radeon_device *rdev, u32 reg);
 void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
 /*
  * Cast helper
  */
 #define to_radeon_fence(p) ((struct radeon_fence *)(p))
 /*
  * Registers read & write functions.
  */
 #define RREG8(reg) readb((rdev->rmmio) + (reg))
 #define WREG8(reg, v) writeb(v, (rdev->rmmio) + (reg))
 #define RREG16(reg) readw((rdev->rmmio) + (reg))
 #define WREG16(reg, v) writew(v, (rdev->rmmio) + (reg))
 #define RREG32(reg) r100_mm_rreg(rdev, (reg))
 #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg)))
 #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v))
 #define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
 #define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
 #define RREG32_PLL(reg) rdev->pll_rreg(rdev, (reg))
 #define WREG32_PLL(reg, v) rdev->pll_wreg(rdev, (reg), (v))
 #define RREG32_MC(reg) rdev->mc_rreg(rdev, (reg))
 #define WREG32_MC(reg, v) rdev->mc_wreg(rdev, (reg), (v))
 #define RREG32_PCIE(reg) rv370_pcie_rreg(rdev, (reg))
 #define WREG32_PCIE(reg, v) rv370_pcie_wreg(rdev, (reg), (v))
 #define RREG32_PCIE_P(reg) rdev->pciep_rreg(rdev, (reg))
 #define WREG32_PCIE_P(reg, v) rdev->pciep_wreg(rdev, (reg), (v))
 #define WREG32_P(reg, val, mask)				\
 	do {							\
 		uint32_t tmp_ = RREG32(reg);			\
 		tmp_ &= (mask);					\
 		tmp_ |= ((val) & ~(mask));			\
 		WREG32(reg, tmp_);				\
 	} while (0)
 #define WREG32_PLL_P(reg, val, mask)				\
 	do {							\
 		uint32_t tmp_ = RREG32_PLL(reg);		\
 		tmp_ &= (mask);					\
 		tmp_ |= ((val) & ~(mask));			\
 		WREG32_PLL(reg, tmp_);				\
 	} while (0)
 #define DREG32_SYS(sqf, rdev, reg) seq_printf((sqf), #reg " : 0x%08X\n", r100_mm_rreg((rdev), (reg)))
 #define RREG32_IO(reg) r100_io_rreg(rdev, (reg))
 #define WREG32_IO(reg, v) r100_io_wreg(rdev, (reg), (v))
 /*
  * Indirect registers accessor
  */
 static inline uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg)
 {
 	uint32_t r;
 	WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask));
 	r = RREG32(RADEON_PCIE_DATA);
 	return r;
 }
 static inline void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
 {
 	WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask));
 	WREG32(RADEON_PCIE_DATA, (v));
 }
 void r100_pll_errata_after_index(struct radeon_device *rdev);
 /*
  * ASICs helpers.
  */
 #define ASIC_IS_RN50(rdev) ((rdev->pdev->device == 0x515e) || \
 			    (rdev->pdev->device == 0x5969))
 #define ASIC_IS_RV100(rdev) ((rdev->family == CHIP_RV100) || \
 		(rdev->family == CHIP_RV200) || \
 		(rdev->family == CHIP_RS100) || \
 		(rdev->family == CHIP_RS200) || \
 		(rdev->family == CHIP_RV250) || \
 		(rdev->family == CHIP_RV280) || \
 		(rdev->family == CHIP_RS300))
 #define ASIC_IS_R300(rdev) ((rdev->family == CHIP_R300)  ||	\
 		(rdev->family == CHIP_RV350) ||			\
 		(rdev->family == CHIP_R350)  ||			\
 		(rdev->family == CHIP_RV380) ||			\
 		(rdev->family == CHIP_R420)  ||			\
 		(rdev->family == CHIP_R423)  ||			\
 		(rdev->family == CHIP_RV410) ||			\
 		(rdev->family == CHIP_RS400) ||			\
 		(rdev->family == CHIP_RS480))
 #define ASIC_IS_X2(rdev) ((rdev->ddev->pdev->device == 0x9441) || \
 		(rdev->ddev->pdev->device == 0x9443) || \
 		(rdev->ddev->pdev->device == 0x944B) || \
 		(rdev->ddev->pdev->device == 0x9506) || \
 		(rdev->ddev->pdev->device == 0x9509) || \
 		(rdev->ddev->pdev->device == 0x950F) || \
 		(rdev->ddev->pdev->device == 0x689C) || \
 		(rdev->ddev->pdev->device == 0x689D))
 #define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
 #define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600)  ||	\
 			    (rdev->family == CHIP_RS690)  ||	\
 			    (rdev->family == CHIP_RS740)  ||	\
 			    (rdev->family >= CHIP_R600))
 #define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
 #define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
 #define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
 #define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
 			     (rdev->flags & RADEON_IS_IGP))
 #define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
 /*
  * BIOS helpers.
  */
 #define RBIOS8(i) (rdev->bios[i])
 #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
 #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
 int radeon_combios_init(struct radeon_device *rdev);
 void radeon_combios_fini(struct radeon_device *rdev);
 int radeon_atombios_init(struct radeon_device *rdev);
 void radeon_atombios_fini(struct radeon_device *rdev);
 /*
  * RING helpers.
  */
 #if DRM_DEBUG_CODE == 0
 static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v)
 {
 	rdev->cp.ring[rdev->cp.wptr++] = v;
 	rdev->cp.wptr &= rdev->cp.ptr_mask;
 	rdev->cp.count_dw--;
 	rdev->cp.ring_free_dw--;
 }
 #else
 /* With debugging this is just too big to inline */
 void radeon_ring_write(struct radeon_device *rdev, uint32_t v);
 #endif
 /*
  * ASICs macro.
  */
 #define radeon_init(rdev) (rdev)->asic->init((rdev))
 #define radeon_fini(rdev) (rdev)->asic->fini((rdev))
 #define radeon_resume(rdev) (rdev)->asic->resume((rdev))
 #define radeon_suspend(rdev) (rdev)->asic->suspend((rdev))
 #define radeon_cs_parse(p) rdev->asic->cs_parse((p))
 #define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state))
 #define radeon_gpu_is_lockup(rdev) (rdev)->asic->gpu_is_lockup((rdev))
 #define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev))
 #define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart_tlb_flush((rdev))
 #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart_set_page((rdev), (i), (p))
 #define radeon_cp_commit(rdev) (rdev)->asic->cp_commit((rdev))
 #define radeon_ring_start(rdev) (rdev)->asic->ring_start((rdev))
 #define radeon_ring_test(rdev) (rdev)->asic->ring_test((rdev))
 #define radeon_ring_ib_execute(rdev, ib) (rdev)->asic->ring_ib_execute((rdev), (ib))
 #define radeon_irq_set(rdev) (rdev)->asic->irq_set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq_process((rdev))
 #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->get_vblank_counter((rdev), (crtc))
 #define radeon_fence_ring_emit(rdev, fence) (rdev)->asic->fence_ring_emit((rdev), (fence))
 #define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy_blit((rdev), (s), (d), (np), (f))
 #define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy_dma((rdev), (s), (d), (np), (f))
 #define radeon_copy(rdev, s, d, np, f) (rdev)->asic->copy((rdev), (s), (d), (np), (f))
 #define radeon_get_engine_clock(rdev) (rdev)->asic->get_engine_clock((rdev))
 #define radeon_set_engine_clock(rdev, e) (rdev)->asic->set_engine_clock((rdev), (e))
 #define radeon_get_memory_clock(rdev) (rdev)->asic->get_memory_clock((rdev))
 #define radeon_set_memory_clock(rdev, e) (rdev)->asic->set_memory_clock((rdev), (e))
 #define radeon_get_pcie_lanes(rdev) (rdev)->asic->get_pcie_lanes((rdev))
 #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->set_pcie_lanes((rdev), (l))
 #define radeon_set_clock_gating(rdev, e) (rdev)->asic->set_clock_gating((rdev), (e))
 #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->set_surface_reg((rdev), (r), (f), (p), (o), (s)))
 #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->clear_surface_reg((rdev), (r)))
 #define radeon_bandwidth_update(rdev) (rdev)->asic->bandwidth_update((rdev))
 #define radeon_hpd_init(rdev) (rdev)->asic->hpd_init((rdev))
 #define radeon_hpd_fini(rdev) (rdev)->asic->hpd_fini((rdev))
 #define radeon_hpd_sense(rdev, hpd) (rdev)->asic->hpd_sense((rdev), (hpd))
 #define radeon_hpd_set_polarity(rdev, hpd) (rdev)->asic->hpd_set_polarity((rdev), (hpd))
 #define radeon_gui_idle(rdev) (rdev)->asic->gui_idle((rdev))
 #define radeon_pm_misc(rdev) (rdev)->asic->pm_misc((rdev))
 #define radeon_pm_prepare(rdev) (rdev)->asic->pm_prepare((rdev))
 #define radeon_pm_finish(rdev) (rdev)->asic->pm_finish((rdev))
 #define radeon_pm_init_profile(rdev) (rdev)->asic->pm_init_profile((rdev))
 #define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm_get_dynpm_state((rdev))
 #define radeon_pre_page_flip(rdev, crtc) rdev->asic->pre_page_flip((rdev), (crtc))
 #define radeon_page_flip(rdev, crtc, base) rdev->asic->page_flip((rdev), (crtc), (base))
 #define radeon_post_page_flip(rdev, crtc) rdev->asic->post_page_flip((rdev), (crtc))
 /* Common functions */
 /* AGP */
 extern int radeon_gpu_reset(struct radeon_device *rdev);
 extern void radeon_agp_disable(struct radeon_device *rdev);
 extern int radeon_modeset_init(struct radeon_device *rdev);
 extern void radeon_modeset_fini(struct radeon_device *rdev);
 extern bool radeon_card_posted(struct radeon_device *rdev);
 extern void radeon_update_bandwidth_info(struct radeon_device *rdev);
 extern void radeon_update_display_priority(struct radeon_device *rdev);
 extern bool radeon_boot_test_post_card(struct radeon_device *rdev);
 extern void radeon_scratch_init(struct radeon_device *rdev);
 extern void radeon_wb_fini(struct radeon_device *rdev);
 extern int radeon_wb_init(struct radeon_device *rdev);
 extern void radeon_wb_disable(struct radeon_device *rdev);
 extern void radeon_surface_init(struct radeon_device *rdev);
 extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data);
 extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable);
 extern void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable);
 extern void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain);
 extern bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo);
 extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base);
 extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 extern int radeon_resume_kms(struct drm_device *dev);
 extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
 extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size);
 /*
  * R600 vram scratch functions
  */
 int r600_vram_scratch_init(struct radeon_device *rdev);
 void r600_vram_scratch_fini(struct radeon_device *rdev);
 /*
  * r600 functions used by radeon_encoder.c
  */
 extern void r600_hdmi_enable(struct drm_encoder *encoder);
 extern void r600_hdmi_disable(struct drm_encoder *encoder);
 extern void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode);
 extern int ni_init_microcode(struct radeon_device *rdev);
 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 /* radeon_acpi.c */
 #if defined(CONFIG_ACPI)
 extern int radeon_acpi_init(struct radeon_device *rdev);
 #else
 static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; }
 #endif
 #include "radeon_object.h"
 #endif

drivers/gpu/drm/radeon/radeon_cs.c

Diff comments View file @ 7a1619b

1	/*	1	/*
2	* Copyright 2008 Jerome Glisse.	2	* Copyright 2008 Jerome Glisse.
3	* All Rights Reserved.	3	* All Rights Reserved.
4	*	4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a	5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),	6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation	7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the	9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:	10	* Software is furnished to do so, subject to the following conditions:
11	*	11	*
12	* The above copyright notice and this permission notice (including the next	12	* The above copyright notice and this permission notice (including the next
13	* paragraph) shall be included in all copies or substantial portions of the	13	* paragraph) shall be included in all copies or substantial portions of the
14	* Software.	14	* Software.
15	*	15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR	19	* PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,	20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER	21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22	* DEALINGS IN THE SOFTWARE.	22	* DEALINGS IN THE SOFTWARE.
23	*	23	*
24	* Authors:	24	* Authors:
25	* Jerome Glisse <glisse@freedesktop.org>	25	* Jerome Glisse <glisse@freedesktop.org>
26	*/	26	*/
27	#include "drmP.h"	27	#include "drmP.h"
28	#include "radeon_drm.h"	28	#include "radeon_drm.h"
29	#include "radeon_reg.h"	29	#include "radeon_reg.h"
30	#include "radeon.h"	30	#include "radeon.h"
31		31
32	void r100_cs_dump_packet(struct radeon_cs_parser *p,	32	void r100_cs_dump_packet(struct radeon_cs_parser *p,
33	struct radeon_cs_packet *pkt);	33	struct radeon_cs_packet *pkt);
34		34
35	int radeon_cs_parser_relocs(struct radeon_cs_parser *p)	35	int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
36	{	36	{
37	struct drm_device *ddev = p->rdev->ddev;	37	struct drm_device *ddev = p->rdev->ddev;
38	struct radeon_cs_chunk *chunk;	38	struct radeon_cs_chunk *chunk;
39	unsigned i, j;	39	unsigned i, j;
40	bool duplicate;	40	bool duplicate;
41		41
42	if (p->chunk_relocs_idx == -1) {	42	if (p->chunk_relocs_idx == -1) {
43	return 0;	43	return 0;
44	}	44	}
45	chunk = &p->chunks[p->chunk_relocs_idx];	45	chunk = &p->chunks[p->chunk_relocs_idx];
46	/* FIXME: we assume that each relocs use 4 dwords */	46	/* FIXME: we assume that each relocs use 4 dwords */
47	p->nrelocs = chunk->length_dw / 4;	47	p->nrelocs = chunk->length_dw / 4;
48	p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);	48	p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
49	if (p->relocs_ptr == NULL) {	49	if (p->relocs_ptr == NULL) {
50	return -ENOMEM;	50	return -ENOMEM;
51	}	51	}
52	p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);	52	p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
53	if (p->relocs == NULL) {	53	if (p->relocs == NULL) {
54	return -ENOMEM;	54	return -ENOMEM;
55	}	55	}
56	for (i = 0; i < p->nrelocs; i++) {	56	for (i = 0; i < p->nrelocs; i++) {
57	struct drm_radeon_cs_reloc *r;	57	struct drm_radeon_cs_reloc *r;
58		58
59	duplicate = false;	59	duplicate = false;
60	r = (struct drm_radeon_cs_reloc )&chunk->kdata[i4];	60	r = (struct drm_radeon_cs_reloc )&chunk->kdata[i4];
61	for (j = 0; j < p->nrelocs; j++) {	61	for (j = 0; j < p->nrelocs; j++) {
62	if (r->handle == p->relocs[j].handle) {	62	if (r->handle == p->relocs[j].handle) {
63	p->relocs_ptr[i] = &p->relocs[j];	63	p->relocs_ptr[i] = &p->relocs[j];
64	duplicate = true;	64	duplicate = true;
65	break;	65	break;
66	}	66	}
67	}	67	}
68	if (!duplicate) {	68	if (!duplicate) {
69	p->relocs[i].gobj = drm_gem_object_lookup(ddev,	69	p->relocs[i].gobj = drm_gem_object_lookup(ddev,
70	p->filp,	70	p->filp,
71	r->handle);	71	r->handle);
72	if (p->relocs[i].gobj == NULL) {	72	if (p->relocs[i].gobj == NULL) {
73	DRM_ERROR("gem object lookup failed 0x%x\n",	73	DRM_ERROR("gem object lookup failed 0x%x\n",
74	r->handle);	74	r->handle);
75	return -ENOENT;	75	return -ENOENT;
76	}	76	}
77	p->relocs_ptr[i] = &p->relocs[i];	77	p->relocs_ptr[i] = &p->relocs[i];
78	p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);	78	p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
79	p->relocs[i].lobj.bo = p->relocs[i].robj;	79	p->relocs[i].lobj.bo = p->relocs[i].robj;
80	p->relocs[i].lobj.wdomain = r->write_domain;	80	p->relocs[i].lobj.wdomain = r->write_domain;
81	p->relocs[i].lobj.rdomain = r->read_domains;	81	p->relocs[i].lobj.rdomain = r->read_domains;
82	p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;	82	p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
83	p->relocs[i].handle = r->handle;	83	p->relocs[i].handle = r->handle;
84	p->relocs[i].flags = r->flags;	84	p->relocs[i].flags = r->flags;
85	radeon_bo_list_add_object(&p->relocs[i].lobj,	85	radeon_bo_list_add_object(&p->relocs[i].lobj,
86	&p->validated);	86	&p->validated);
87	}	87	}
88	}	88	}
89	return radeon_bo_list_validate(&p->validated);	89	return radeon_bo_list_validate(&p->validated);
90	}	90	}
91		91
92	int radeon_cs_parser_init(struct radeon_cs_parser p, void data)	92	int radeon_cs_parser_init(struct radeon_cs_parser p, void data)
93	{	93	{
94	struct drm_radeon_cs *cs = data;	94	struct drm_radeon_cs *cs = data;
95	uint64_t *chunk_array_ptr;	95	uint64_t *chunk_array_ptr;
96	unsigned size, i;	96	unsigned size, i;
97		97
98	if (!cs->num_chunks) {	98	if (!cs->num_chunks) {
99	return 0;	99	return 0;
100	}	100	}
101	/* get chunks */	101	/* get chunks */
102	INIT_LIST_HEAD(&p->validated);	102	INIT_LIST_HEAD(&p->validated);
103	p->idx = 0;	103	p->idx = 0;
104	p->chunk_ib_idx = -1;	104	p->chunk_ib_idx = -1;
105	p->chunk_relocs_idx = -1;	105	p->chunk_relocs_idx = -1;
106	p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);	106	p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
107	if (p->chunks_array == NULL) {	107	if (p->chunks_array == NULL) {
108	return -ENOMEM;	108	return -ENOMEM;
109	}	109	}
110	chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);	110	chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
111	if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,	111	if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,
112	sizeof(uint64_t)*cs->num_chunks)) {	112	sizeof(uint64_t)*cs->num_chunks)) {
113	return -EFAULT;	113	return -EFAULT;
114	}	114	}
115	p->nchunks = cs->num_chunks;	115	p->nchunks = cs->num_chunks;
116	p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);	116	p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
117	if (p->chunks == NULL) {	117	if (p->chunks == NULL) {
118	return -ENOMEM;	118	return -ENOMEM;
119	}	119	}
120	for (i = 0; i < p->nchunks; i++) {	120	for (i = 0; i < p->nchunks; i++) {
121	struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;	121	struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
122	struct drm_radeon_cs_chunk user_chunk;	122	struct drm_radeon_cs_chunk user_chunk;
123	uint32_t __user *cdata;	123	uint32_t __user *cdata;
124		124
125	chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];	125	chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
126	if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,	126	if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,
127	sizeof(struct drm_radeon_cs_chunk))) {	127	sizeof(struct drm_radeon_cs_chunk))) {
128	return -EFAULT;	128	return -EFAULT;
129	}	129	}
130	p->chunks[i].length_dw = user_chunk.length_dw;	130	p->chunks[i].length_dw = user_chunk.length_dw;
131	p->chunks[i].kdata = NULL;	131	p->chunks[i].kdata = NULL;
132	p->chunks[i].chunk_id = user_chunk.chunk_id;	132	p->chunks[i].chunk_id = user_chunk.chunk_id;
133		133
134	if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {	134	if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
135	p->chunk_relocs_idx = i;	135	p->chunk_relocs_idx = i;
136	}	136	}
137	if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {	137	if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
138	p->chunk_ib_idx = i;	138	p->chunk_ib_idx = i;
139	/* zero length IB isn't useful */	139	/* zero length IB isn't useful */
140	if (p->chunks[i].length_dw == 0)	140	if (p->chunks[i].length_dw == 0)
141	return -EINVAL;	141	return -EINVAL;
142	}	142	}
143		143
144	p->chunks[i].length_dw = user_chunk.length_dw;	144	p->chunks[i].length_dw = user_chunk.length_dw;
145	p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;	145	p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
146		146
147	cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;	147	cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
148	if (p->chunks[i].chunk_id != RADEON_CHUNK_ID_IB) {	148	if (p->chunks[i].chunk_id != RADEON_CHUNK_ID_IB) {
149	size = p->chunks[i].length_dw * sizeof(uint32_t);	149	size = p->chunks[i].length_dw * sizeof(uint32_t);
150	p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);	150	p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
151	if (p->chunks[i].kdata == NULL) {	151	if (p->chunks[i].kdata == NULL) {
152	return -ENOMEM;	152	return -ENOMEM;
153	}	153	}
154	if (DRM_COPY_FROM_USER(p->chunks[i].kdata,	154	if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
155	p->chunks[i].user_ptr, size)) {	155	p->chunks[i].user_ptr, size)) {
156	return -EFAULT;	156	return -EFAULT;
157	}	157	}
158	} else {	158	} else {
159	p->chunks[i].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);	159	p->chunks[i].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
160	p->chunks[i].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);	160	p->chunks[i].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
161	if (p->chunks[i].kpage[0] == NULL \|\| p->chunks[i].kpage[1] == NULL) {	161	if (p->chunks[i].kpage[0] == NULL \|\| p->chunks[i].kpage[1] == NULL) {
162	kfree(p->chunks[i].kpage[0]);	162	kfree(p->chunks[i].kpage[0]);
163	kfree(p->chunks[i].kpage[1]);	163	kfree(p->chunks[i].kpage[1]);
164	return -ENOMEM;	164	return -ENOMEM;
165	}	165	}
166	p->chunks[i].kpage_idx[0] = -1;	166	p->chunks[i].kpage_idx[0] = -1;
167	p->chunks[i].kpage_idx[1] = -1;	167	p->chunks[i].kpage_idx[1] = -1;
168	p->chunks[i].last_copied_page = -1;	168	p->chunks[i].last_copied_page = -1;
169	p->chunks[i].last_page_index = ((p->chunks[i].length_dw * 4) - 1) / PAGE_SIZE;	169	p->chunks[i].last_page_index = ((p->chunks[i].length_dw * 4) - 1) / PAGE_SIZE;
170	}	170	}
171	}	171	}
172	if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {	172	if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
173	DRM_ERROR("cs IB too big: %d\n",	173	DRM_ERROR("cs IB too big: %d\n",
174	p->chunks[p->chunk_ib_idx].length_dw);	174	p->chunks[p->chunk_ib_idx].length_dw);
175	return -EINVAL;	175	return -EINVAL;
176	}	176	}
177	return 0;	177	return 0;
178	}	178	}
179		179
180	/**	180	/**
181	* cs_parser_fini() - clean parser states	181	* cs_parser_fini() - clean parser states
182	* @parser: parser structure holding parsing context.	182	* @parser: parser structure holding parsing context.
183	* @error: error number	183	* @error: error number
184	*	184	*
185	* If error is set than unvalidate buffer, otherwise just free memory	185	* If error is set than unvalidate buffer, otherwise just free memory
186	* used by parsing context.	186	* used by parsing context.
187	**/	187	**/
188	static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)	188	static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
189	{	189	{
190	unsigned i;	190	unsigned i;
191		191
192		192
193	if (!error && parser->ib)	193	if (!error && parser->ib)
194	ttm_eu_fence_buffer_objects(&parser->validated,	194	ttm_eu_fence_buffer_objects(&parser->validated,
195	parser->ib->fence);	195	parser->ib->fence);
196	else	196	else
197	ttm_eu_backoff_reservation(&parser->validated);	197	ttm_eu_backoff_reservation(&parser->validated);
198		198
199	if (parser->relocs != NULL) {	199	if (parser->relocs != NULL) {
200	for (i = 0; i < parser->nrelocs; i++) {	200	for (i = 0; i < parser->nrelocs; i++) {
201	if (parser->relocs[i].gobj)	201	if (parser->relocs[i].gobj)
202	drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);	202	drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
203	}	203	}
204	}	204	}
205	kfree(parser->track);	205	kfree(parser->track);
206	kfree(parser->relocs);	206	kfree(parser->relocs);
207	kfree(parser->relocs_ptr);	207	kfree(parser->relocs_ptr);
208	for (i = 0; i < parser->nchunks; i++) {	208	for (i = 0; i < parser->nchunks; i++) {
209	kfree(parser->chunks[i].kdata);	209	kfree(parser->chunks[i].kdata);
210	kfree(parser->chunks[i].kpage[0]);	210	kfree(parser->chunks[i].kpage[0]);
211	kfree(parser->chunks[i].kpage[1]);	211	kfree(parser->chunks[i].kpage[1]);
212	}	212	}
213	kfree(parser->chunks);	213	kfree(parser->chunks);
214	kfree(parser->chunks_array);	214	kfree(parser->chunks_array);
215	radeon_ib_free(parser->rdev, &parser->ib);	215	radeon_ib_free(parser->rdev, &parser->ib);
216	}	216	}
217		217
218	int radeon_cs_ioctl(struct drm_device dev, void data, struct drm_file *filp)	218	int radeon_cs_ioctl(struct drm_device dev, void data, struct drm_file *filp)
219	{	219	{
220	struct radeon_device *rdev = dev->dev_private;	220	struct radeon_device *rdev = dev->dev_private;
221	struct radeon_cs_parser parser;	221	struct radeon_cs_parser parser;
222	struct radeon_cs_chunk *ib_chunk;	222	struct radeon_cs_chunk *ib_chunk;
223	int r;	223	int r;
224		224
225	mutex_lock(&rdev->cs_mutex);	225	radeon_mutex_lock(&rdev->cs_mutex);
226	/* initialize parser */	226	/* initialize parser */
227	memset(&parser, 0, sizeof(struct radeon_cs_parser));	227	memset(&parser, 0, sizeof(struct radeon_cs_parser));
228	parser.filp = filp;	228	parser.filp = filp;
229	parser.rdev = rdev;	229	parser.rdev = rdev;
230	parser.dev = rdev->dev;	230	parser.dev = rdev->dev;
231	parser.family = rdev->family;	231	parser.family = rdev->family;
232	r = radeon_cs_parser_init(&parser, data);	232	r = radeon_cs_parser_init(&parser, data);
233	if (r) {	233	if (r) {
234	DRM_ERROR("Failed to initialize parser !\n");	234	DRM_ERROR("Failed to initialize parser !\n");
235	radeon_cs_parser_fini(&parser, r);	235	radeon_cs_parser_fini(&parser, r);
236	mutex_unlock(&rdev->cs_mutex);	236	radeon_mutex_unlock(&rdev->cs_mutex);
237	return r;	237	return r;
238	}	238	}
239	r = radeon_ib_get(rdev, &parser.ib);	239	r = radeon_ib_get(rdev, &parser.ib);
240	if (r) {	240	if (r) {
241	DRM_ERROR("Failed to get ib !\n");	241	DRM_ERROR("Failed to get ib !\n");
242	radeon_cs_parser_fini(&parser, r);	242	radeon_cs_parser_fini(&parser, r);
243	mutex_unlock(&rdev->cs_mutex);	243	radeon_mutex_unlock(&rdev->cs_mutex);
244	return r;	244	return r;
245	}	245	}
246	r = radeon_cs_parser_relocs(&parser);	246	r = radeon_cs_parser_relocs(&parser);
247	if (r) {	247	if (r) {
248	if (r != -ERESTARTSYS)	248	if (r != -ERESTARTSYS)
249	DRM_ERROR("Failed to parse relocation %d!\n", r);	249	DRM_ERROR("Failed to parse relocation %d!\n", r);
250	radeon_cs_parser_fini(&parser, r);	250	radeon_cs_parser_fini(&parser, r);
251	mutex_unlock(&rdev->cs_mutex);	251	radeon_mutex_unlock(&rdev->cs_mutex);
252	return r;	252	return r;
253	}	253	}
254	/* Copy the packet into the IB, the parser will read from the	254	/* Copy the packet into the IB, the parser will read from the
255	* input memory (cached) and write to the IB (which can be	255	* input memory (cached) and write to the IB (which can be
256	* uncached). */	256	* uncached). */
257	ib_chunk = &parser.chunks[parser.chunk_ib_idx];	257	ib_chunk = &parser.chunks[parser.chunk_ib_idx];
258	parser.ib->length_dw = ib_chunk->length_dw;	258	parser.ib->length_dw = ib_chunk->length_dw;
259	r = radeon_cs_parse(&parser);	259	r = radeon_cs_parse(&parser);
260	if (r \|\| parser.parser_error) {	260	if (r \|\| parser.parser_error) {
261	DRM_ERROR("Invalid command stream !\n");	261	DRM_ERROR("Invalid command stream !\n");
262	radeon_cs_parser_fini(&parser, r);	262	radeon_cs_parser_fini(&parser, r);
263	mutex_unlock(&rdev->cs_mutex);	263	radeon_mutex_unlock(&rdev->cs_mutex);
264	return r;	264	return r;
265	}	265	}
266	r = radeon_cs_finish_pages(&parser);	266	r = radeon_cs_finish_pages(&parser);
267	if (r) {	267	if (r) {
268	DRM_ERROR("Invalid command stream !\n");	268	DRM_ERROR("Invalid command stream !\n");
269	radeon_cs_parser_fini(&parser, r);	269	radeon_cs_parser_fini(&parser, r);
270	mutex_unlock(&rdev->cs_mutex);	270	radeon_mutex_unlock(&rdev->cs_mutex);
271	return r;	271	return r;
272	}	272	}
273	r = radeon_ib_schedule(rdev, parser.ib);	273	r = radeon_ib_schedule(rdev, parser.ib);
274	if (r) {	274	if (r) {
275	DRM_ERROR("Failed to schedule IB !\n");	275	DRM_ERROR("Failed to schedule IB !\n");
276	}	276	}
277	radeon_cs_parser_fini(&parser, r);	277	radeon_cs_parser_fini(&parser, r);
278	mutex_unlock(&rdev->cs_mutex);	278	radeon_mutex_unlock(&rdev->cs_mutex);
279	return r;	279	return r;
280	}	280	}
281		281
282	int radeon_cs_finish_pages(struct radeon_cs_parser *p)	282	int radeon_cs_finish_pages(struct radeon_cs_parser *p)
283	{	283	{
284	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];	284	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
285	int i;	285	int i;
286	int size = PAGE_SIZE;	286	int size = PAGE_SIZE;
287		287
288	for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {	288	for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
289	if (i == ibc->last_page_index) {	289	if (i == ibc->last_page_index) {
290	size = (ibc->length_dw * 4) % PAGE_SIZE;	290	size = (ibc->length_dw * 4) % PAGE_SIZE;
291	if (size == 0)	291	if (size == 0)
292	size = PAGE_SIZE;	292	size = PAGE_SIZE;
293	}	293	}
294		294
295	if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),	295	if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),
296	ibc->user_ptr + (i * PAGE_SIZE),	296	ibc->user_ptr + (i * PAGE_SIZE),
297	size))	297	size))
298	return -EFAULT;	298	return -EFAULT;
299	}	299	}
300	return 0;	300	return 0;
301	}	301	}
302		302
303	int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)	303	int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
304	{	304	{
305	int new_page;	305	int new_page;
306	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];	306	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
307	int i;	307	int i;
308	int size = PAGE_SIZE;	308	int size = PAGE_SIZE;
309		309
310	for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {	310	for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
311	if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),	311	if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),
312	ibc->user_ptr + (i * PAGE_SIZE),	312	ibc->user_ptr + (i * PAGE_SIZE),
313	PAGE_SIZE)) {	313	PAGE_SIZE)) {
314	p->parser_error = -EFAULT;	314	p->parser_error = -EFAULT;
315	return 0;	315	return 0;
316	}	316	}
317	}	317	}
318		318
319	new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;	319	new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
320		320
321	if (pg_idx == ibc->last_page_index) {	321	if (pg_idx == ibc->last_page_index) {
322	size = (ibc->length_dw * 4) % PAGE_SIZE;	322	size = (ibc->length_dw * 4) % PAGE_SIZE;
323	if (size == 0)	323	if (size == 0)
324	size = PAGE_SIZE;	324	size = PAGE_SIZE;
325	}	325	}
326		326
327	if (DRM_COPY_FROM_USER(ibc->kpage[new_page],	327	if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
328	ibc->user_ptr + (pg_idx * PAGE_SIZE),	328	ibc->user_ptr + (pg_idx * PAGE_SIZE),
329	size)) {	329	size)) {
330	p->parser_error = -EFAULT;	330	p->parser_error = -EFAULT;
331	return 0;	331	return 0;
332	}	332	}
333		333
334	/* copy to IB here */	334	/* copy to IB here */
335	memcpy((void )(p->ib->ptr+(pg_idx(PAGE_SIZE/4))), ibc->kpage[new_page], size);	335	memcpy((void )(p->ib->ptr+(pg_idx(PAGE_SIZE/4))), ibc->kpage[new_page], size);
336		336
337	ibc->last_copied_page = pg_idx;	337	ibc->last_copied_page = pg_idx;
338	ibc->kpage_idx[new_page] = pg_idx;	338	ibc->kpage_idx[new_page] = pg_idx;
339		339
340	return new_page;	340	return new_page;
341	}	341	}
342		342

drivers/gpu/drm/radeon/radeon_device.c

Diff comments View file @ 7a1619b

1	/*	1	/*
2	* Copyright 2008 Advanced Micro Devices, Inc.	2	* Copyright 2008 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.	3	* Copyright 2008 Red Hat Inc.
4	* Copyright 2009 Jerome Glisse.	4	* Copyright 2009 Jerome Glisse.
5	*	5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a	6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),	7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation	8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,	9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the	10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:	11	* Software is furnished to do so, subject to the following conditions:
12	*	12	*
13	* The above copyright notice and this permission notice shall be included in	13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.	14	* all copies or substantial portions of the Software.
15	*	15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR	19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,	20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR	21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22	* OTHER DEALINGS IN THE SOFTWARE.	22	* OTHER DEALINGS IN THE SOFTWARE.
23	*	23	*
24	* Authors: Dave Airlie	24	* Authors: Dave Airlie
25	* Alex Deucher	25	* Alex Deucher
26	* Jerome Glisse	26	* Jerome Glisse
27	*/	27	*/
28	#include <linux/console.h>	28	#include <linux/console.h>
29	#include <linux/slab.h>	29	#include <linux/slab.h>
30	#include <drm/drmP.h>	30	#include <drm/drmP.h>
31	#include <drm/drm_crtc_helper.h>	31	#include <drm/drm_crtc_helper.h>
32	#include <drm/radeon_drm.h>	32	#include <drm/radeon_drm.h>
33	#include <linux/vgaarb.h>	33	#include <linux/vgaarb.h>
34	#include <linux/vga_switcheroo.h>	34	#include <linux/vga_switcheroo.h>
35	#include <linux/efi.h>	35	#include <linux/efi.h>
36	#include "radeon_reg.h"	36	#include "radeon_reg.h"
37	#include "radeon.h"	37	#include "radeon.h"
38	#include "atom.h"	38	#include "atom.h"
39		39
40	static const char radeon_family_name[][16] = {	40	static const char radeon_family_name[][16] = {
41	"R100",	41	"R100",
42	"RV100",	42	"RV100",
43	"RS100",	43	"RS100",
44	"RV200",	44	"RV200",
45	"RS200",	45	"RS200",
46	"R200",	46	"R200",
47	"RV250",	47	"RV250",
48	"RS300",	48	"RS300",
49	"RV280",	49	"RV280",
50	"R300",	50	"R300",
51	"R350",	51	"R350",
52	"RV350",	52	"RV350",
53	"RV380",	53	"RV380",
54	"R420",	54	"R420",
55	"R423",	55	"R423",
56	"RV410",	56	"RV410",
57	"RS400",	57	"RS400",
58	"RS480",	58	"RS480",
59	"RS600",	59	"RS600",
60	"RS690",	60	"RS690",
61	"RS740",	61	"RS740",
62	"RV515",	62	"RV515",
63	"R520",	63	"R520",
64	"RV530",	64	"RV530",
65	"RV560",	65	"RV560",
66	"RV570",	66	"RV570",
67	"R580",	67	"R580",
68	"R600",	68	"R600",
69	"RV610",	69	"RV610",
70	"RV630",	70	"RV630",
71	"RV670",	71	"RV670",
72	"RV620",	72	"RV620",
73	"RV635",	73	"RV635",
74	"RS780",	74	"RS780",
75	"RS880",	75	"RS880",
76	"RV770",	76	"RV770",
77	"RV730",	77	"RV730",
78	"RV710",	78	"RV710",
79	"RV740",	79	"RV740",
80	"CEDAR",	80	"CEDAR",
81	"REDWOOD",	81	"REDWOOD",
82	"JUNIPER",	82	"JUNIPER",
83	"CYPRESS",	83	"CYPRESS",
84	"HEMLOCK",	84	"HEMLOCK",
85	"PALM",	85	"PALM",
86	"SUMO",	86	"SUMO",
87	"SUMO2",	87	"SUMO2",
88	"BARTS",	88	"BARTS",
89	"TURKS",	89	"TURKS",
90	"CAICOS",	90	"CAICOS",
91	"CAYMAN",	91	"CAYMAN",
92	"LAST",	92	"LAST",
93	};	93	};
94		94
95	/*	95	/*
96	* Clear GPU surface registers.	96	* Clear GPU surface registers.
97	*/	97	*/
98	void radeon_surface_init(struct radeon_device *rdev)	98	void radeon_surface_init(struct radeon_device *rdev)
99	{	99	{
100	/* FIXME: check this out */	100	/* FIXME: check this out */
101	if (rdev->family < CHIP_R600) {	101	if (rdev->family < CHIP_R600) {
102	int i;	102	int i;
103		103
104	for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {	104	for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
105	if (rdev->surface_regs[i].bo)	105	if (rdev->surface_regs[i].bo)
106	radeon_bo_get_surface_reg(rdev->surface_regs[i].bo);	106	radeon_bo_get_surface_reg(rdev->surface_regs[i].bo);
107	else	107	else
108	radeon_clear_surface_reg(rdev, i);	108	radeon_clear_surface_reg(rdev, i);
109	}	109	}
110	/* enable surfaces */	110	/* enable surfaces */
111	WREG32(RADEON_SURFACE_CNTL, 0);	111	WREG32(RADEON_SURFACE_CNTL, 0);
112	}	112	}
113	}	113	}
114		114
115	/*	115	/*
116	* GPU scratch registers helpers function.	116	* GPU scratch registers helpers function.
117	*/	117	*/
118	void radeon_scratch_init(struct radeon_device *rdev)	118	void radeon_scratch_init(struct radeon_device *rdev)
119	{	119	{
120	int i;	120	int i;
121		121
122	/* FIXME: check this out */	122	/* FIXME: check this out */
123	if (rdev->family < CHIP_R300) {	123	if (rdev->family < CHIP_R300) {
124	rdev->scratch.num_reg = 5;	124	rdev->scratch.num_reg = 5;
125	} else {	125	} else {
126	rdev->scratch.num_reg = 7;	126	rdev->scratch.num_reg = 7;
127	}	127	}
128	rdev->scratch.reg_base = RADEON_SCRATCH_REG0;	128	rdev->scratch.reg_base = RADEON_SCRATCH_REG0;
129	for (i = 0; i < rdev->scratch.num_reg; i++) {	129	for (i = 0; i < rdev->scratch.num_reg; i++) {
130	rdev->scratch.free[i] = true;	130	rdev->scratch.free[i] = true;
131	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);	131	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
132	}	132	}
133	}	133	}
134		134
135	int radeon_scratch_get(struct radeon_device rdev, uint32_t reg)	135	int radeon_scratch_get(struct radeon_device rdev, uint32_t reg)
136	{	136	{
137	int i;	137	int i;
138		138
139	for (i = 0; i < rdev->scratch.num_reg; i++) {	139	for (i = 0; i < rdev->scratch.num_reg; i++) {
140	if (rdev->scratch.free[i]) {	140	if (rdev->scratch.free[i]) {
141	rdev->scratch.free[i] = false;	141	rdev->scratch.free[i] = false;
142	*reg = rdev->scratch.reg[i];	142	*reg = rdev->scratch.reg[i];
143	return 0;	143	return 0;
144	}	144	}
145	}	145	}
146	return -EINVAL;	146	return -EINVAL;
147	}	147	}
148		148
149	void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)	149	void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
150	{	150	{
151	int i;	151	int i;
152		152
153	for (i = 0; i < rdev->scratch.num_reg; i++) {	153	for (i = 0; i < rdev->scratch.num_reg; i++) {
154	if (rdev->scratch.reg[i] == reg) {	154	if (rdev->scratch.reg[i] == reg) {
155	rdev->scratch.free[i] = true;	155	rdev->scratch.free[i] = true;
156	return;	156	return;
157	}	157	}
158	}	158	}
159	}	159	}
160		160
161	void radeon_wb_disable(struct radeon_device *rdev)	161	void radeon_wb_disable(struct radeon_device *rdev)
162	{	162	{
163	int r;	163	int r;
164		164
165	if (rdev->wb.wb_obj) {	165	if (rdev->wb.wb_obj) {
166	r = radeon_bo_reserve(rdev->wb.wb_obj, false);	166	r = radeon_bo_reserve(rdev->wb.wb_obj, false);
167	if (unlikely(r != 0))	167	if (unlikely(r != 0))
168	return;	168	return;
169	radeon_bo_kunmap(rdev->wb.wb_obj);	169	radeon_bo_kunmap(rdev->wb.wb_obj);
170	radeon_bo_unpin(rdev->wb.wb_obj);	170	radeon_bo_unpin(rdev->wb.wb_obj);
171	radeon_bo_unreserve(rdev->wb.wb_obj);	171	radeon_bo_unreserve(rdev->wb.wb_obj);
172	}	172	}
173	rdev->wb.enabled = false;	173	rdev->wb.enabled = false;
174	}	174	}
175		175
176	void radeon_wb_fini(struct radeon_device *rdev)	176	void radeon_wb_fini(struct radeon_device *rdev)
177	{	177	{
178	radeon_wb_disable(rdev);	178	radeon_wb_disable(rdev);
179	if (rdev->wb.wb_obj) {	179	if (rdev->wb.wb_obj) {
180	radeon_bo_unref(&rdev->wb.wb_obj);	180	radeon_bo_unref(&rdev->wb.wb_obj);
181	rdev->wb.wb = NULL;	181	rdev->wb.wb = NULL;
182	rdev->wb.wb_obj = NULL;	182	rdev->wb.wb_obj = NULL;
183	}	183	}
184	}	184	}
185		185
186	int radeon_wb_init(struct radeon_device *rdev)	186	int radeon_wb_init(struct radeon_device *rdev)
187	{	187	{
188	int r;	188	int r;
189		189
190	if (rdev->wb.wb_obj == NULL) {	190	if (rdev->wb.wb_obj == NULL) {
191	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,	191	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
192	RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);	192	RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
193	if (r) {	193	if (r) {
194	dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);	194	dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
195	return r;	195	return r;
196	}	196	}
197	}	197	}
198	r = radeon_bo_reserve(rdev->wb.wb_obj, false);	198	r = radeon_bo_reserve(rdev->wb.wb_obj, false);
199	if (unlikely(r != 0)) {	199	if (unlikely(r != 0)) {
200	radeon_wb_fini(rdev);	200	radeon_wb_fini(rdev);
201	return r;	201	return r;
202	}	202	}
203	r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,	203	r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
204	&rdev->wb.gpu_addr);	204	&rdev->wb.gpu_addr);
205	if (r) {	205	if (r) {
206	radeon_bo_unreserve(rdev->wb.wb_obj);	206	radeon_bo_unreserve(rdev->wb.wb_obj);
207	dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);	207	dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
208	radeon_wb_fini(rdev);	208	radeon_wb_fini(rdev);
209	return r;	209	return r;
210	}	210	}
211	r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);	211	r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
212	radeon_bo_unreserve(rdev->wb.wb_obj);	212	radeon_bo_unreserve(rdev->wb.wb_obj);
213	if (r) {	213	if (r) {
214	dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);	214	dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
215	radeon_wb_fini(rdev);	215	radeon_wb_fini(rdev);
216	return r;	216	return r;
217	}	217	}
218		218
219	/* clear wb memory */	219	/* clear wb memory */
220	memset((char *)rdev->wb.wb, 0, RADEON_GPU_PAGE_SIZE);	220	memset((char *)rdev->wb.wb, 0, RADEON_GPU_PAGE_SIZE);
221	/* disable event_write fences */	221	/* disable event_write fences */
222	rdev->wb.use_event = false;	222	rdev->wb.use_event = false;
223	/* disabled via module param */	223	/* disabled via module param */
224	if (radeon_no_wb == 1)	224	if (radeon_no_wb == 1)
225	rdev->wb.enabled = false;	225	rdev->wb.enabled = false;
226	else {	226	else {
227	/* often unreliable on AGP */	227	/* often unreliable on AGP */
228	if (rdev->flags & RADEON_IS_AGP) {	228	if (rdev->flags & RADEON_IS_AGP) {
229	rdev->wb.enabled = false;	229	rdev->wb.enabled = false;
230	} else {	230	} else {
231	rdev->wb.enabled = true;	231	rdev->wb.enabled = true;
232	/* event_write fences are only available on r600+ */	232	/* event_write fences are only available on r600+ */
233	if (rdev->family >= CHIP_R600)	233	if (rdev->family >= CHIP_R600)
234	rdev->wb.use_event = true;	234	rdev->wb.use_event = true;
235	}	235	}
236	}	236	}
237	/* always use writeback/events on NI */	237	/* always use writeback/events on NI */
238	if (ASIC_IS_DCE5(rdev)) {	238	if (ASIC_IS_DCE5(rdev)) {
239	rdev->wb.enabled = true;	239	rdev->wb.enabled = true;
240	rdev->wb.use_event = true;	240	rdev->wb.use_event = true;
241	}	241	}
242		242
243	dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");	243	dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
244		244
245	return 0;	245	return 0;
246	}	246	}
247		247
248	/**	248	/**
249	* radeon_vram_location - try to find VRAM location	249	* radeon_vram_location - try to find VRAM location
250	* @rdev: radeon device structure holding all necessary informations	250	* @rdev: radeon device structure holding all necessary informations
251	* @mc: memory controller structure holding memory informations	251	* @mc: memory controller structure holding memory informations
252	* @base: base address at which to put VRAM	252	* @base: base address at which to put VRAM
253	*	253	*
254	* Function will place try to place VRAM at base address provided	254	* Function will place try to place VRAM at base address provided
255	* as parameter (which is so far either PCI aperture address or	255	* as parameter (which is so far either PCI aperture address or
256	* for IGP TOM base address).	256	* for IGP TOM base address).
257	*	257	*
258	* If there is not enough space to fit the unvisible VRAM in the 32bits	258	* If there is not enough space to fit the unvisible VRAM in the 32bits
259	* address space then we limit the VRAM size to the aperture.	259	* address space then we limit the VRAM size to the aperture.
260	*	260	*
261	* If we are using AGP and if the AGP aperture doesn't allow us to have	261	* If we are using AGP and if the AGP aperture doesn't allow us to have
262	* room for all the VRAM than we restrict the VRAM to the PCI aperture	262	* room for all the VRAM than we restrict the VRAM to the PCI aperture
263	* size and print a warning.	263	* size and print a warning.
264	*	264	*
265	* This function will never fails, worst case are limiting VRAM.	265	* This function will never fails, worst case are limiting VRAM.
266	*	266	*
267	* Note: GTT start, end, size should be initialized before calling this	267	* Note: GTT start, end, size should be initialized before calling this
268	* function on AGP platform.	268	* function on AGP platform.
269	*	269	*
270	* Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,	270	* Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
271	* this shouldn't be a problem as we are using the PCI aperture as a reference.	271	* this shouldn't be a problem as we are using the PCI aperture as a reference.
272	* Otherwise this would be needed for rv280, all r3xx, and all r4xx, but	272	* Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
273	* not IGP.	273	* not IGP.
274	*	274	*
275	* Note: we use mc_vram_size as on some board we need to program the mc to	275	* Note: we use mc_vram_size as on some board we need to program the mc to
276	* cover the whole aperture even if VRAM size is inferior to aperture size	276	* cover the whole aperture even if VRAM size is inferior to aperture size
277	* Novell bug 204882 + along with lots of ubuntu ones	277	* Novell bug 204882 + along with lots of ubuntu ones
278	*	278	*
279	* Note: when limiting vram it's safe to overwritte real_vram_size because	279	* Note: when limiting vram it's safe to overwritte real_vram_size because
280	* we are not in case where real_vram_size is inferior to mc_vram_size (ie	280	* we are not in case where real_vram_size is inferior to mc_vram_size (ie
281	* note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu	281	* note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
282	* ones)	282	* ones)
283	*	283	*
284	* Note: IGP TOM addr should be the same as the aperture addr, we don't	284	* Note: IGP TOM addr should be the same as the aperture addr, we don't
285	* explicitly check for that thought.	285	* explicitly check for that thought.
286	*	286	*
287	* FIXME: when reducing VRAM size align new size on power of 2.	287	* FIXME: when reducing VRAM size align new size on power of 2.
288	*/	288	*/
289	void radeon_vram_location(struct radeon_device rdev, struct radeon_mc mc, u64 base)	289	void radeon_vram_location(struct radeon_device rdev, struct radeon_mc mc, u64 base)
290	{	290	{
291	mc->vram_start = base;	291	mc->vram_start = base;
292	if (mc->mc_vram_size > (0xFFFFFFFF - base + 1)) {	292	if (mc->mc_vram_size > (0xFFFFFFFF - base + 1)) {
293	dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");	293	dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
294	mc->real_vram_size = mc->aper_size;	294	mc->real_vram_size = mc->aper_size;
295	mc->mc_vram_size = mc->aper_size;	295	mc->mc_vram_size = mc->aper_size;
296	}	296	}
297	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;	297	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
298	if (rdev->flags & RADEON_IS_AGP && mc->vram_end > mc->gtt_start && mc->vram_start <= mc->gtt_end) {	298	if (rdev->flags & RADEON_IS_AGP && mc->vram_end > mc->gtt_start && mc->vram_start <= mc->gtt_end) {
299	dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");	299	dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
300	mc->real_vram_size = mc->aper_size;	300	mc->real_vram_size = mc->aper_size;
301	mc->mc_vram_size = mc->aper_size;	301	mc->mc_vram_size = mc->aper_size;
302	}	302	}
303	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;	303	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
304	if (radeon_vram_limit && radeon_vram_limit < mc->real_vram_size)	304	if (radeon_vram_limit && radeon_vram_limit < mc->real_vram_size)
305	mc->real_vram_size = radeon_vram_limit;	305	mc->real_vram_size = radeon_vram_limit;
306	dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",	306	dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
307	mc->mc_vram_size >> 20, mc->vram_start,	307	mc->mc_vram_size >> 20, mc->vram_start,
308	mc->vram_end, mc->real_vram_size >> 20);	308	mc->vram_end, mc->real_vram_size >> 20);
309	}	309	}
310		310
311	/**	311	/**
312	* radeon_gtt_location - try to find GTT location	312	* radeon_gtt_location - try to find GTT location
313	* @rdev: radeon device structure holding all necessary informations	313	* @rdev: radeon device structure holding all necessary informations
314	* @mc: memory controller structure holding memory informations	314	* @mc: memory controller structure holding memory informations
315	*	315	*
316	* Function will place try to place GTT before or after VRAM.	316	* Function will place try to place GTT before or after VRAM.
317	*	317	*
318	* If GTT size is bigger than space left then we ajust GTT size.	318	* If GTT size is bigger than space left then we ajust GTT size.
319	* Thus function will never fails.	319	* Thus function will never fails.
320	*	320	*
321	* FIXME: when reducing GTT size align new size on power of 2.	321	* FIXME: when reducing GTT size align new size on power of 2.
322	*/	322	*/
323	void radeon_gtt_location(struct radeon_device rdev, struct radeon_mc mc)	323	void radeon_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
324	{	324	{
325	u64 size_af, size_bf;	325	u64 size_af, size_bf;
326		326
327	size_af = ((0xFFFFFFFF - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;	327	size_af = ((0xFFFFFFFF - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
328	size_bf = mc->vram_start & ~mc->gtt_base_align;	328	size_bf = mc->vram_start & ~mc->gtt_base_align;
329	if (size_bf > size_af) {	329	if (size_bf > size_af) {
330	if (mc->gtt_size > size_bf) {	330	if (mc->gtt_size > size_bf) {
331	dev_warn(rdev->dev, "limiting GTT\n");	331	dev_warn(rdev->dev, "limiting GTT\n");
332	mc->gtt_size = size_bf;	332	mc->gtt_size = size_bf;
333	}	333	}
334	mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;	334	mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
335	} else {	335	} else {
336	if (mc->gtt_size > size_af) {	336	if (mc->gtt_size > size_af) {
337	dev_warn(rdev->dev, "limiting GTT\n");	337	dev_warn(rdev->dev, "limiting GTT\n");
338	mc->gtt_size = size_af;	338	mc->gtt_size = size_af;
339	}	339	}
340	mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;	340	mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
341	}	341	}
342	mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;	342	mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
343	dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",	343	dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
344	mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);	344	mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
345	}	345	}
346		346
347	/*	347	/*
348	* GPU helpers function.	348	* GPU helpers function.
349	*/	349	*/
350	bool radeon_card_posted(struct radeon_device *rdev)	350	bool radeon_card_posted(struct radeon_device *rdev)
351	{	351	{
352	uint32_t reg;	352	uint32_t reg;
353		353
354	if (efi_enabled && rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE)	354	if (efi_enabled && rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE)
355	return false;	355	return false;
356		356
357	/* first check CRTCs */	357	/* first check CRTCs */
358	if (ASIC_IS_DCE41(rdev)) {	358	if (ASIC_IS_DCE41(rdev)) {
359	reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) \|	359	reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) \|
360	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);	360	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
361	if (reg & EVERGREEN_CRTC_MASTER_EN)	361	if (reg & EVERGREEN_CRTC_MASTER_EN)
362	return true;	362	return true;
363	} else if (ASIC_IS_DCE4(rdev)) {	363	} else if (ASIC_IS_DCE4(rdev)) {
364	reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) \|	364	reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) \|
365	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) \|	365	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) \|
366	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) \|	366	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) \|
367	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) \|	367	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) \|
368	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) \|	368	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) \|
369	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);	369	RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
370	if (reg & EVERGREEN_CRTC_MASTER_EN)	370	if (reg & EVERGREEN_CRTC_MASTER_EN)
371	return true;	371	return true;
372	} else if (ASIC_IS_AVIVO(rdev)) {	372	} else if (ASIC_IS_AVIVO(rdev)) {
373	reg = RREG32(AVIVO_D1CRTC_CONTROL) \|	373	reg = RREG32(AVIVO_D1CRTC_CONTROL) \|
374	RREG32(AVIVO_D2CRTC_CONTROL);	374	RREG32(AVIVO_D2CRTC_CONTROL);
375	if (reg & AVIVO_CRTC_EN) {	375	if (reg & AVIVO_CRTC_EN) {
376	return true;	376	return true;
377	}	377	}
378	} else {	378	} else {
379	reg = RREG32(RADEON_CRTC_GEN_CNTL) \|	379	reg = RREG32(RADEON_CRTC_GEN_CNTL) \|
380	RREG32(RADEON_CRTC2_GEN_CNTL);	380	RREG32(RADEON_CRTC2_GEN_CNTL);
381	if (reg & RADEON_CRTC_EN) {	381	if (reg & RADEON_CRTC_EN) {
382	return true;	382	return true;
383	}	383	}
384	}	384	}
385		385
386	/* then check MEM_SIZE, in case the crtcs are off */	386	/* then check MEM_SIZE, in case the crtcs are off */
387	if (rdev->family >= CHIP_R600)	387	if (rdev->family >= CHIP_R600)
388	reg = RREG32(R600_CONFIG_MEMSIZE);	388	reg = RREG32(R600_CONFIG_MEMSIZE);
389	else	389	else
390	reg = RREG32(RADEON_CONFIG_MEMSIZE);	390	reg = RREG32(RADEON_CONFIG_MEMSIZE);
391		391
392	if (reg)	392	if (reg)
393	return true;	393	return true;
394		394
395	return false;	395	return false;
396		396
397	}	397	}
398		398
399	void radeon_update_bandwidth_info(struct radeon_device *rdev)	399	void radeon_update_bandwidth_info(struct radeon_device *rdev)
400	{	400	{
401	fixed20_12 a;	401	fixed20_12 a;
402	u32 sclk = rdev->pm.current_sclk;	402	u32 sclk = rdev->pm.current_sclk;
403	u32 mclk = rdev->pm.current_mclk;	403	u32 mclk = rdev->pm.current_mclk;
404		404
405	/* sclk/mclk in Mhz */	405	/* sclk/mclk in Mhz */
406	a.full = dfixed_const(100);	406	a.full = dfixed_const(100);
407	rdev->pm.sclk.full = dfixed_const(sclk);	407	rdev->pm.sclk.full = dfixed_const(sclk);
408	rdev->pm.sclk.full = dfixed_div(rdev->pm.sclk, a);	408	rdev->pm.sclk.full = dfixed_div(rdev->pm.sclk, a);
409	rdev->pm.mclk.full = dfixed_const(mclk);	409	rdev->pm.mclk.full = dfixed_const(mclk);
410	rdev->pm.mclk.full = dfixed_div(rdev->pm.mclk, a);	410	rdev->pm.mclk.full = dfixed_div(rdev->pm.mclk, a);
411		411
412	if (rdev->flags & RADEON_IS_IGP) {	412	if (rdev->flags & RADEON_IS_IGP) {
413	a.full = dfixed_const(16);	413	a.full = dfixed_const(16);
414	/* core_bandwidth = sclk(Mhz) * 16 */	414	/* core_bandwidth = sclk(Mhz) * 16 */
415	rdev->pm.core_bandwidth.full = dfixed_div(rdev->pm.sclk, a);	415	rdev->pm.core_bandwidth.full = dfixed_div(rdev->pm.sclk, a);
416	}	416	}
417	}	417	}
418		418
419	bool radeon_boot_test_post_card(struct radeon_device *rdev)	419	bool radeon_boot_test_post_card(struct radeon_device *rdev)
420	{	420	{
421	if (radeon_card_posted(rdev))	421	if (radeon_card_posted(rdev))
422	return true;	422	return true;
423		423
424	if (rdev->bios) {	424	if (rdev->bios) {
425	DRM_INFO("GPU not posted. posting now...\n");	425	DRM_INFO("GPU not posted. posting now...\n");
426	if (rdev->is_atom_bios)	426	if (rdev->is_atom_bios)
427	atom_asic_init(rdev->mode_info.atom_context);	427	atom_asic_init(rdev->mode_info.atom_context);
428	else	428	else
429	radeon_combios_asic_init(rdev->ddev);	429	radeon_combios_asic_init(rdev->ddev);
430	return true;	430	return true;
431	} else {	431	} else {
432	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");	432	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
433	return false;	433	return false;
434	}	434	}
435	}	435	}
436		436
437	int radeon_dummy_page_init(struct radeon_device *rdev)	437	int radeon_dummy_page_init(struct radeon_device *rdev)
438	{	438	{
439	if (rdev->dummy_page.page)	439	if (rdev->dummy_page.page)
440	return 0;	440	return 0;
441	rdev->dummy_page.page = alloc_page(GFP_DMA32 \| GFP_KERNEL \| __GFP_ZERO);	441	rdev->dummy_page.page = alloc_page(GFP_DMA32 \| GFP_KERNEL \| __GFP_ZERO);
442	if (rdev->dummy_page.page == NULL)	442	if (rdev->dummy_page.page == NULL)
443	return -ENOMEM;	443	return -ENOMEM;
444	rdev->dummy_page.addr = pci_map_page(rdev->pdev, rdev->dummy_page.page,	444	rdev->dummy_page.addr = pci_map_page(rdev->pdev, rdev->dummy_page.page,
445	0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);	445	0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
446	if (pci_dma_mapping_error(rdev->pdev, rdev->dummy_page.addr)) {	446	if (pci_dma_mapping_error(rdev->pdev, rdev->dummy_page.addr)) {
447	dev_err(&rdev->pdev->dev, "Failed to DMA MAP the dummy page\n");	447	dev_err(&rdev->pdev->dev, "Failed to DMA MAP the dummy page\n");
448	__free_page(rdev->dummy_page.page);	448	__free_page(rdev->dummy_page.page);
449	rdev->dummy_page.page = NULL;	449	rdev->dummy_page.page = NULL;
450	return -ENOMEM;	450	return -ENOMEM;
451	}	451	}
452	return 0;	452	return 0;
453	}	453	}
454		454
455	void radeon_dummy_page_fini(struct radeon_device *rdev)	455	void radeon_dummy_page_fini(struct radeon_device *rdev)
456	{	456	{
457	if (rdev->dummy_page.page == NULL)	457	if (rdev->dummy_page.page == NULL)
458	return;	458	return;
459	pci_unmap_page(rdev->pdev, rdev->dummy_page.addr,	459	pci_unmap_page(rdev->pdev, rdev->dummy_page.addr,
460	PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);	460	PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
461	__free_page(rdev->dummy_page.page);	461	__free_page(rdev->dummy_page.page);
462	rdev->dummy_page.page = NULL;	462	rdev->dummy_page.page = NULL;
463	}	463	}
464		464
465		465
466	/* ATOM accessor methods */	466	/* ATOM accessor methods */
467	static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)	467	static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
468	{	468	{
469	struct radeon_device *rdev = info->dev->dev_private;	469	struct radeon_device *rdev = info->dev->dev_private;
470	uint32_t r;	470	uint32_t r;
471		471
472	r = rdev->pll_rreg(rdev, reg);	472	r = rdev->pll_rreg(rdev, reg);
473	return r;	473	return r;
474	}	474	}
475		475
476	static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)	476	static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
477	{	477	{
478	struct radeon_device *rdev = info->dev->dev_private;	478	struct radeon_device *rdev = info->dev->dev_private;
479		479
480	rdev->pll_wreg(rdev, reg, val);	480	rdev->pll_wreg(rdev, reg, val);
481	}	481	}
482		482
483	static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)	483	static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
484	{	484	{
485	struct radeon_device *rdev = info->dev->dev_private;	485	struct radeon_device *rdev = info->dev->dev_private;
486	uint32_t r;	486	uint32_t r;
487		487
488	r = rdev->mc_rreg(rdev, reg);	488	r = rdev->mc_rreg(rdev, reg);
489	return r;	489	return r;
490	}	490	}
491		491
492	static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)	492	static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
493	{	493	{
494	struct radeon_device *rdev = info->dev->dev_private;	494	struct radeon_device *rdev = info->dev->dev_private;
495		495
496	rdev->mc_wreg(rdev, reg, val);	496	rdev->mc_wreg(rdev, reg, val);
497	}	497	}
498		498
499	static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)	499	static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
500	{	500	{
501	struct radeon_device *rdev = info->dev->dev_private;	501	struct radeon_device *rdev = info->dev->dev_private;
502		502
503	WREG32(reg*4, val);	503	WREG32(reg*4, val);
504	}	504	}
505		505
506	static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)	506	static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
507	{	507	{
508	struct radeon_device *rdev = info->dev->dev_private;	508	struct radeon_device *rdev = info->dev->dev_private;
509	uint32_t r;	509	uint32_t r;
510		510
511	r = RREG32(reg*4);	511	r = RREG32(reg*4);
512	return r;	512	return r;
513	}	513	}
514		514
515	static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)	515	static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
516	{	516	{
517	struct radeon_device *rdev = info->dev->dev_private;	517	struct radeon_device *rdev = info->dev->dev_private;
518		518
519	WREG32_IO(reg*4, val);	519	WREG32_IO(reg*4, val);
520	}	520	}
521		521
522	static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)	522	static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
523	{	523	{
524	struct radeon_device *rdev = info->dev->dev_private;	524	struct radeon_device *rdev = info->dev->dev_private;
525	uint32_t r;	525	uint32_t r;
526		526
527	r = RREG32_IO(reg*4);	527	r = RREG32_IO(reg*4);
528	return r;	528	return r;
529	}	529	}
530		530
531	int radeon_atombios_init(struct radeon_device *rdev)	531	int radeon_atombios_init(struct radeon_device *rdev)
532	{	532	{
533	struct card_info *atom_card_info =	533	struct card_info *atom_card_info =
534	kzalloc(sizeof(struct card_info), GFP_KERNEL);	534	kzalloc(sizeof(struct card_info), GFP_KERNEL);
535		535
536	if (!atom_card_info)	536	if (!atom_card_info)
537	return -ENOMEM;	537	return -ENOMEM;
538		538
539	rdev->mode_info.atom_card_info = atom_card_info;	539	rdev->mode_info.atom_card_info = atom_card_info;
540	atom_card_info->dev = rdev->ddev;	540	atom_card_info->dev = rdev->ddev;
541	atom_card_info->reg_read = cail_reg_read;	541	atom_card_info->reg_read = cail_reg_read;
542	atom_card_info->reg_write = cail_reg_write;	542	atom_card_info->reg_write = cail_reg_write;
543	/* needed for iio ops */	543	/* needed for iio ops */
544	if (rdev->rio_mem) {	544	if (rdev->rio_mem) {
545	atom_card_info->ioreg_read = cail_ioreg_read;	545	atom_card_info->ioreg_read = cail_ioreg_read;
546	atom_card_info->ioreg_write = cail_ioreg_write;	546	atom_card_info->ioreg_write = cail_ioreg_write;
547	} else {	547	} else {
548	DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");	548	DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");
549	atom_card_info->ioreg_read = cail_reg_read;	549	atom_card_info->ioreg_read = cail_reg_read;
550	atom_card_info->ioreg_write = cail_reg_write;	550	atom_card_info->ioreg_write = cail_reg_write;
551	}	551	}
552	atom_card_info->mc_read = cail_mc_read;	552	atom_card_info->mc_read = cail_mc_read;
553	atom_card_info->mc_write = cail_mc_write;	553	atom_card_info->mc_write = cail_mc_write;
554	atom_card_info->pll_read = cail_pll_read;	554	atom_card_info->pll_read = cail_pll_read;
555	atom_card_info->pll_write = cail_pll_write;	555	atom_card_info->pll_write = cail_pll_write;
556		556
557	rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios);	557	rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios);
558	mutex_init(&rdev->mode_info.atom_context->mutex);	558	mutex_init(&rdev->mode_info.atom_context->mutex);
559	radeon_atom_initialize_bios_scratch_regs(rdev->ddev);	559	radeon_atom_initialize_bios_scratch_regs(rdev->ddev);
560	atom_allocate_fb_scratch(rdev->mode_info.atom_context);	560	atom_allocate_fb_scratch(rdev->mode_info.atom_context);
561	return 0;	561	return 0;
562	}	562	}
563		563
564	void radeon_atombios_fini(struct radeon_device *rdev)	564	void radeon_atombios_fini(struct radeon_device *rdev)
565	{	565	{
566	if (rdev->mode_info.atom_context) {	566	if (rdev->mode_info.atom_context) {
567	kfree(rdev->mode_info.atom_context->scratch);	567	kfree(rdev->mode_info.atom_context->scratch);
568	kfree(rdev->mode_info.atom_context);	568	kfree(rdev->mode_info.atom_context);
569	}	569	}
570	kfree(rdev->mode_info.atom_card_info);	570	kfree(rdev->mode_info.atom_card_info);
571	}	571	}
572		572
573	int radeon_combios_init(struct radeon_device *rdev)	573	int radeon_combios_init(struct radeon_device *rdev)
574	{	574	{
575	radeon_combios_initialize_bios_scratch_regs(rdev->ddev);	575	radeon_combios_initialize_bios_scratch_regs(rdev->ddev);
576	return 0;	576	return 0;
577	}	577	}
578		578
579	void radeon_combios_fini(struct radeon_device *rdev)	579	void radeon_combios_fini(struct radeon_device *rdev)
580	{	580	{
581	}	581	}
582		582
583	/* if we get transitioned to only one device, tak VGA back */	583	/* if we get transitioned to only one device, tak VGA back */
584	static unsigned int radeon_vga_set_decode(void *cookie, bool state)	584	static unsigned int radeon_vga_set_decode(void *cookie, bool state)
585	{	585	{
586	struct radeon_device *rdev = cookie;	586	struct radeon_device *rdev = cookie;
587	radeon_vga_set_state(rdev, state);	587	radeon_vga_set_state(rdev, state);
588	if (state)	588	if (state)
589	return VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM \|	589	return VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM \|
590	VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;	590	VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
591	else	591	else
592	return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;	592	return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
593	}	593	}
594		594
595	void radeon_check_arguments(struct radeon_device *rdev)	595	void radeon_check_arguments(struct radeon_device *rdev)
596	{	596	{
597	/* vramlimit must be a power of two */	597	/* vramlimit must be a power of two */
598	switch (radeon_vram_limit) {	598	switch (radeon_vram_limit) {
599	case 0:	599	case 0:
600	case 4:	600	case 4:
601	case 8:	601	case 8:
602	case 16:	602	case 16:
603	case 32:	603	case 32:
604	case 64:	604	case 64:
605	case 128:	605	case 128:
606	case 256:	606	case 256:
607	case 512:	607	case 512:
608	case 1024:	608	case 1024:
609	case 2048:	609	case 2048:
610	case 4096:	610	case 4096:
611	break;	611	break;
612	default:	612	default:
613	dev_warn(rdev->dev, "vram limit (%d) must be a power of 2\n",	613	dev_warn(rdev->dev, "vram limit (%d) must be a power of 2\n",
614	radeon_vram_limit);	614	radeon_vram_limit);
615	radeon_vram_limit = 0;	615	radeon_vram_limit = 0;
616	break;	616	break;
617	}	617	}
618	radeon_vram_limit = radeon_vram_limit << 20;	618	radeon_vram_limit = radeon_vram_limit << 20;
619	/* gtt size must be power of two and greater or equal to 32M */	619	/* gtt size must be power of two and greater or equal to 32M */
620	switch (radeon_gart_size) {	620	switch (radeon_gart_size) {
621	case 4:	621	case 4:
622	case 8:	622	case 8:
623	case 16:	623	case 16:
624	dev_warn(rdev->dev, "gart size (%d) too small forcing to 512M\n",	624	dev_warn(rdev->dev, "gart size (%d) too small forcing to 512M\n",
625	radeon_gart_size);	625	radeon_gart_size);
626	radeon_gart_size = 512;	626	radeon_gart_size = 512;
627	break;	627	break;
628	case 32:	628	case 32:
629	case 64:	629	case 64:
630	case 128:	630	case 128:
631	case 256:	631	case 256:
632	case 512:	632	case 512:
633	case 1024:	633	case 1024:
634	case 2048:	634	case 2048:
635	case 4096:	635	case 4096:
636	break;	636	break;
637	default:	637	default:
638	dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n",	638	dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n",
639	radeon_gart_size);	639	radeon_gart_size);
640	radeon_gart_size = 512;	640	radeon_gart_size = 512;
641	break;	641	break;
642	}	642	}
643	rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024;	643	rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024;
644	/* AGP mode can only be -1, 1, 2, 4, 8 */	644	/* AGP mode can only be -1, 1, 2, 4, 8 */
645	switch (radeon_agpmode) {	645	switch (radeon_agpmode) {
646	case -1:	646	case -1:
647	case 0:	647	case 0:
648	case 1:	648	case 1:
649	case 2:	649	case 2:
650	case 4:	650	case 4:
651	case 8:	651	case 8:
652	break;	652	break;
653	default:	653	default:
654	dev_warn(rdev->dev, "invalid AGP mode %d (valid mode: "	654	dev_warn(rdev->dev, "invalid AGP mode %d (valid mode: "
655	"-1, 0, 1, 2, 4, 8)\n", radeon_agpmode);	655	"-1, 0, 1, 2, 4, 8)\n", radeon_agpmode);
656	radeon_agpmode = 0;	656	radeon_agpmode = 0;
657	break;	657	break;
658	}	658	}
659	}	659	}
660		660
661	static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)	661	static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
662	{	662	{
663	struct drm_device *dev = pci_get_drvdata(pdev);	663	struct drm_device *dev = pci_get_drvdata(pdev);
664	pm_message_t pmm = { .event = PM_EVENT_SUSPEND };	664	pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
665	if (state == VGA_SWITCHEROO_ON) {	665	if (state == VGA_SWITCHEROO_ON) {
666	printk(KERN_INFO "radeon: switched on\n");	666	printk(KERN_INFO "radeon: switched on\n");
667	/* don't suspend or resume card normally */	667	/* don't suspend or resume card normally */
668	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;	668	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
669	radeon_resume_kms(dev);	669	radeon_resume_kms(dev);
670	dev->switch_power_state = DRM_SWITCH_POWER_ON;	670	dev->switch_power_state = DRM_SWITCH_POWER_ON;
671	drm_kms_helper_poll_enable(dev);	671	drm_kms_helper_poll_enable(dev);
672	} else {	672	} else {
673	printk(KERN_INFO "radeon: switched off\n");	673	printk(KERN_INFO "radeon: switched off\n");
674	drm_kms_helper_poll_disable(dev);	674	drm_kms_helper_poll_disable(dev);
675	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;	675	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
676	radeon_suspend_kms(dev, pmm);	676	radeon_suspend_kms(dev, pmm);
677	dev->switch_power_state = DRM_SWITCH_POWER_OFF;	677	dev->switch_power_state = DRM_SWITCH_POWER_OFF;
678	}	678	}
679	}	679	}
680		680
681	static bool radeon_switcheroo_can_switch(struct pci_dev *pdev)	681	static bool radeon_switcheroo_can_switch(struct pci_dev *pdev)
682	{	682	{
683	struct drm_device *dev = pci_get_drvdata(pdev);	683	struct drm_device *dev = pci_get_drvdata(pdev);
684	bool can_switch;	684	bool can_switch;
685		685
686	spin_lock(&dev->count_lock);	686	spin_lock(&dev->count_lock);
687	can_switch = (dev->open_count == 0);	687	can_switch = (dev->open_count == 0);
688	spin_unlock(&dev->count_lock);	688	spin_unlock(&dev->count_lock);
689	return can_switch;	689	return can_switch;
690	}	690	}
691		691
692		692
693	int radeon_device_init(struct radeon_device *rdev,	693	int radeon_device_init(struct radeon_device *rdev,
694	struct drm_device *ddev,	694	struct drm_device *ddev,
695	struct pci_dev *pdev,	695	struct pci_dev *pdev,
696	uint32_t flags)	696	uint32_t flags)
697	{	697	{
698	int r, i;	698	int r, i;
699	int dma_bits;	699	int dma_bits;
700		700
701	rdev->shutdown = false;	701	rdev->shutdown = false;
702	rdev->dev = &pdev->dev;	702	rdev->dev = &pdev->dev;
703	rdev->ddev = ddev;	703	rdev->ddev = ddev;
704	rdev->pdev = pdev;	704	rdev->pdev = pdev;
705	rdev->flags = flags;	705	rdev->flags = flags;
706	rdev->family = flags & RADEON_FAMILY_MASK;	706	rdev->family = flags & RADEON_FAMILY_MASK;
707	rdev->is_atom_bios = false;	707	rdev->is_atom_bios = false;
708	rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT;	708	rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT;
709	rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024;	709	rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024;
710	rdev->gpu_lockup = false;	710	rdev->gpu_lockup = false;
711	rdev->accel_working = false;	711	rdev->accel_working = false;
712		712
713	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n",	713	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n",
714	radeon_family_name[rdev->family], pdev->vendor, pdev->device,	714	radeon_family_name[rdev->family], pdev->vendor, pdev->device,
715	pdev->subsystem_vendor, pdev->subsystem_device);	715	pdev->subsystem_vendor, pdev->subsystem_device);
716		716
717	/* mutex initialization are all done here so we	717	/* mutex initialization are all done here so we
718	* can recall function without having locking issues */	718	* can recall function without having locking issues */
719	mutex_init(&rdev->cs_mutex);	719	radeon_mutex_init(&rdev->cs_mutex);
720	mutex_init(&rdev->ib_pool.mutex);	720	mutex_init(&rdev->ib_pool.mutex);
721	mutex_init(&rdev->cp.mutex);	721	mutex_init(&rdev->cp.mutex);
722	mutex_init(&rdev->dc_hw_i2c_mutex);	722	mutex_init(&rdev->dc_hw_i2c_mutex);
723	if (rdev->family >= CHIP_R600)	723	if (rdev->family >= CHIP_R600)
724	spin_lock_init(&rdev->ih.lock);	724	spin_lock_init(&rdev->ih.lock);
725	mutex_init(&rdev->gem.mutex);	725	mutex_init(&rdev->gem.mutex);
726	mutex_init(&rdev->pm.mutex);	726	mutex_init(&rdev->pm.mutex);
727	mutex_init(&rdev->vram_mutex);	727	mutex_init(&rdev->vram_mutex);
728	rwlock_init(&rdev->fence_drv.lock);	728	rwlock_init(&rdev->fence_drv.lock);
729	INIT_LIST_HEAD(&rdev->gem.objects);	729	INIT_LIST_HEAD(&rdev->gem.objects);
730	init_waitqueue_head(&rdev->irq.vblank_queue);	730	init_waitqueue_head(&rdev->irq.vblank_queue);
731	init_waitqueue_head(&rdev->irq.idle_queue);	731	init_waitqueue_head(&rdev->irq.idle_queue);
732		732
733	/* Set asic functions */	733	/* Set asic functions */
734	r = radeon_asic_init(rdev);	734	r = radeon_asic_init(rdev);
735	if (r)	735	if (r)
736	return r;	736	return r;
737	radeon_check_arguments(rdev);	737	radeon_check_arguments(rdev);
738		738
739	/* all of the newer IGP chips have an internal gart	739	/* all of the newer IGP chips have an internal gart
740	* However some rs4xx report as AGP, so remove that here.	740	* However some rs4xx report as AGP, so remove that here.
741	*/	741	*/
742	if ((rdev->family >= CHIP_RS400) &&	742	if ((rdev->family >= CHIP_RS400) &&
743	(rdev->flags & RADEON_IS_IGP)) {	743	(rdev->flags & RADEON_IS_IGP)) {
744	rdev->flags &= ~RADEON_IS_AGP;	744	rdev->flags &= ~RADEON_IS_AGP;
745	}	745	}
746		746
747	if (rdev->flags & RADEON_IS_AGP && radeon_agpmode == -1) {	747	if (rdev->flags & RADEON_IS_AGP && radeon_agpmode == -1) {
748	radeon_agp_disable(rdev);	748	radeon_agp_disable(rdev);
749	}	749	}
750		750
751	/* set DMA mask + need_dma32 flags.	751	/* set DMA mask + need_dma32 flags.
752	* PCIE - can handle 40-bits.	752	* PCIE - can handle 40-bits.
753	* IGP - can handle 40-bits	753	* IGP - can handle 40-bits
754	* AGP - generally dma32 is safest	754	* AGP - generally dma32 is safest
755	* PCI - dma32 for legacy pci gart, 40 bits on newer asics	755	* PCI - dma32 for legacy pci gart, 40 bits on newer asics
756	*/	756	*/
757	rdev->need_dma32 = false;	757	rdev->need_dma32 = false;
758	if (rdev->flags & RADEON_IS_AGP)	758	if (rdev->flags & RADEON_IS_AGP)
759	rdev->need_dma32 = true;	759	rdev->need_dma32 = true;
760	if ((rdev->flags & RADEON_IS_PCI) &&	760	if ((rdev->flags & RADEON_IS_PCI) &&
761	(rdev->family < CHIP_RS400))	761	(rdev->family < CHIP_RS400))
762	rdev->need_dma32 = true;	762	rdev->need_dma32 = true;
763		763
764	dma_bits = rdev->need_dma32 ? 32 : 40;	764	dma_bits = rdev->need_dma32 ? 32 : 40;
765	r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));	765	r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
766	if (r) {	766	if (r) {
767	rdev->need_dma32 = true;	767	rdev->need_dma32 = true;
768	printk(KERN_WARNING "radeon: No suitable DMA available.\n");	768	printk(KERN_WARNING "radeon: No suitable DMA available.\n");
769	}	769	}
770		770
771	/* Registers mapping */	771	/* Registers mapping */
772	/* TODO: block userspace mapping of io register */	772	/* TODO: block userspace mapping of io register */
773	rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);	773	rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);
774	rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);	774	rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);
775	rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size);	775	rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size);
776	if (rdev->rmmio == NULL) {	776	if (rdev->rmmio == NULL) {
777	return -ENOMEM;	777	return -ENOMEM;
778	}	778	}
779	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)rdev->rmmio_base);	779	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)rdev->rmmio_base);
780	DRM_INFO("register mmio size: %u\n", (unsigned)rdev->rmmio_size);	780	DRM_INFO("register mmio size: %u\n", (unsigned)rdev->rmmio_size);
781		781
782	/* io port mapping */	782	/* io port mapping */
783	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {	783	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
784	if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) {	784	if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) {
785	rdev->rio_mem_size = pci_resource_len(rdev->pdev, i);	785	rdev->rio_mem_size = pci_resource_len(rdev->pdev, i);
786	rdev->rio_mem = pci_iomap(rdev->pdev, i, rdev->rio_mem_size);	786	rdev->rio_mem = pci_iomap(rdev->pdev, i, rdev->rio_mem_size);
787	break;	787	break;
788	}	788	}
789	}	789	}
790	if (rdev->rio_mem == NULL)	790	if (rdev->rio_mem == NULL)
791	DRM_ERROR("Unable to find PCI I/O BAR\n");	791	DRM_ERROR("Unable to find PCI I/O BAR\n");
792		792
793	/* if we have > 1 VGA cards, then disable the radeon VGA resources */	793	/* if we have > 1 VGA cards, then disable the radeon VGA resources */
794	/* this will fail for cards that aren't VGA class devices, just	794	/* this will fail for cards that aren't VGA class devices, just
795	* ignore it */	795	* ignore it */
796	vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);	796	vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
797	vga_switcheroo_register_client(rdev->pdev,	797	vga_switcheroo_register_client(rdev->pdev,
798	radeon_switcheroo_set_state,	798	radeon_switcheroo_set_state,
799	NULL,	799	NULL,
800	radeon_switcheroo_can_switch);	800	radeon_switcheroo_can_switch);
801		801
802	r = radeon_init(rdev);	802	r = radeon_init(rdev);
803	if (r)	803	if (r)
804	return r;	804	return r;
805		805
806	if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) {	806	if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) {
807	/* Acceleration not working on AGP card try again	807	/* Acceleration not working on AGP card try again
808	* with fallback to PCI or PCIE GART	808	* with fallback to PCI or PCIE GART
809	*/	809	*/
810	radeon_asic_reset(rdev);	810	radeon_asic_reset(rdev);
811	radeon_fini(rdev);	811	radeon_fini(rdev);
812	radeon_agp_disable(rdev);	812	radeon_agp_disable(rdev);
813	r = radeon_init(rdev);	813	r = radeon_init(rdev);
814	if (r)	814	if (r)
815	return r;	815	return r;
816	}	816	}
817	if (radeon_testing) {	817	if (radeon_testing) {
818	radeon_test_moves(rdev);	818	radeon_test_moves(rdev);
819	}	819	}
820	if (radeon_benchmarking) {	820	if (radeon_benchmarking) {
821	radeon_benchmark(rdev, radeon_benchmarking);	821	radeon_benchmark(rdev, radeon_benchmarking);
822	}	822	}
823	return 0;	823	return 0;
824	}	824	}
825		825
826	void radeon_device_fini(struct radeon_device *rdev)	826	void radeon_device_fini(struct radeon_device *rdev)
827	{	827	{
828	DRM_INFO("radeon: finishing device.\n");	828	DRM_INFO("radeon: finishing device.\n");
829	rdev->shutdown = true;	829	rdev->shutdown = true;
830	/* evict vram memory */	830	/* evict vram memory */
831	radeon_bo_evict_vram(rdev);	831	radeon_bo_evict_vram(rdev);
832	radeon_fini(rdev);	832	radeon_fini(rdev);
833	vga_switcheroo_unregister_client(rdev->pdev);	833	vga_switcheroo_unregister_client(rdev->pdev);
834	vga_client_register(rdev->pdev, NULL, NULL, NULL);	834	vga_client_register(rdev->pdev, NULL, NULL, NULL);
835	if (rdev->rio_mem)	835	if (rdev->rio_mem)
836	pci_iounmap(rdev->pdev, rdev->rio_mem);	836	pci_iounmap(rdev->pdev, rdev->rio_mem);
837	rdev->rio_mem = NULL;	837	rdev->rio_mem = NULL;
838	iounmap(rdev->rmmio);	838	iounmap(rdev->rmmio);
839	rdev->rmmio = NULL;	839	rdev->rmmio = NULL;
840	}	840	}
841		841
842		842
843	/*	843	/*
844	* Suspend & resume.	844	* Suspend & resume.
845	*/	845	*/
846	int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)	846	int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
847	{	847	{
848	struct radeon_device *rdev;	848	struct radeon_device *rdev;
849	struct drm_crtc *crtc;	849	struct drm_crtc *crtc;
850	struct drm_connector *connector;	850	struct drm_connector *connector;
851	int r;	851	int r;
852		852
853	if (dev == NULL \|\| dev->dev_private == NULL) {	853	if (dev == NULL \|\| dev->dev_private == NULL) {
854	return -ENODEV;	854	return -ENODEV;
855	}	855	}
856	if (state.event == PM_EVENT_PRETHAW) {	856	if (state.event == PM_EVENT_PRETHAW) {
857	return 0;	857	return 0;
858	}	858	}
859	rdev = dev->dev_private;	859	rdev = dev->dev_private;
860		860
861	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)	861	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
862	return 0;	862	return 0;
863		863
864	/* turn off display hw */	864	/* turn off display hw */
865	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {	865	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
866	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);	866	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
867	}	867	}
868		868
869	/* unpin the front buffers */	869	/* unpin the front buffers */
870	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {	870	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
871	struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->fb);	871	struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->fb);
872	struct radeon_bo *robj;	872	struct radeon_bo *robj;
873		873
874	if (rfb == NULL \|\| rfb->obj == NULL) {	874	if (rfb == NULL \|\| rfb->obj == NULL) {
875	continue;	875	continue;
876	}	876	}
877	robj = gem_to_radeon_bo(rfb->obj);	877	robj = gem_to_radeon_bo(rfb->obj);
878	/* don't unpin kernel fb objects */	878	/* don't unpin kernel fb objects */
879	if (!radeon_fbdev_robj_is_fb(rdev, robj)) {	879	if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
880	r = radeon_bo_reserve(robj, false);	880	r = radeon_bo_reserve(robj, false);
881	if (r == 0) {	881	if (r == 0) {
882	radeon_bo_unpin(robj);	882	radeon_bo_unpin(robj);
883	radeon_bo_unreserve(robj);	883	radeon_bo_unreserve(robj);
884	}	884	}
885	}	885	}
886	}	886	}
887	/* evict vram memory */	887	/* evict vram memory */
888	radeon_bo_evict_vram(rdev);	888	radeon_bo_evict_vram(rdev);
889	/* wait for gpu to finish processing current batch */	889	/* wait for gpu to finish processing current batch */
890	radeon_fence_wait_last(rdev);	890	radeon_fence_wait_last(rdev);
891		891
892	radeon_save_bios_scratch_regs(rdev);	892	radeon_save_bios_scratch_regs(rdev);
893		893
894	radeon_pm_suspend(rdev);	894	radeon_pm_suspend(rdev);
895	radeon_suspend(rdev);	895	radeon_suspend(rdev);
896	radeon_hpd_fini(rdev);	896	radeon_hpd_fini(rdev);
897	/* evict remaining vram memory */	897	/* evict remaining vram memory */
898	radeon_bo_evict_vram(rdev);	898	radeon_bo_evict_vram(rdev);
899		899
900	radeon_agp_suspend(rdev);	900	radeon_agp_suspend(rdev);
901		901
902	pci_save_state(dev->pdev);	902	pci_save_state(dev->pdev);
903	if (state.event == PM_EVENT_SUSPEND) {	903	if (state.event == PM_EVENT_SUSPEND) {
904	/* Shut down the device */	904	/* Shut down the device */
905	pci_disable_device(dev->pdev);	905	pci_disable_device(dev->pdev);
906	pci_set_power_state(dev->pdev, PCI_D3hot);	906	pci_set_power_state(dev->pdev, PCI_D3hot);
907	}	907	}
908	console_lock();	908	console_lock();
909	radeon_fbdev_set_suspend(rdev, 1);	909	radeon_fbdev_set_suspend(rdev, 1);
910	console_unlock();	910	console_unlock();
911	return 0;	911	return 0;
912	}	912	}
913		913
914	int radeon_resume_kms(struct drm_device *dev)	914	int radeon_resume_kms(struct drm_device *dev)
915	{	915	{
916	struct drm_connector *connector;	916	struct drm_connector *connector;
917	struct radeon_device *rdev = dev->dev_private;	917	struct radeon_device *rdev = dev->dev_private;
918		918
919	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)	919	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
920	return 0;	920	return 0;
921		921
922	console_lock();	922	console_lock();
923	pci_set_power_state(dev->pdev, PCI_D0);	923	pci_set_power_state(dev->pdev, PCI_D0);
924	pci_restore_state(dev->pdev);	924	pci_restore_state(dev->pdev);
925	if (pci_enable_device(dev->pdev)) {	925	if (pci_enable_device(dev->pdev)) {
926	console_unlock();	926	console_unlock();
927	return -1;	927	return -1;
928	}	928	}
929	pci_set_master(dev->pdev);	929	pci_set_master(dev->pdev);
930	/* resume AGP if in use */	930	/* resume AGP if in use */
931	radeon_agp_resume(rdev);	931	radeon_agp_resume(rdev);
932	radeon_resume(rdev);	932	radeon_resume(rdev);
933	radeon_pm_resume(rdev);	933	radeon_pm_resume(rdev);
934	radeon_restore_bios_scratch_regs(rdev);	934	radeon_restore_bios_scratch_regs(rdev);
935		935
936	radeon_fbdev_set_suspend(rdev, 0);	936	radeon_fbdev_set_suspend(rdev, 0);
937	console_unlock();	937	console_unlock();
938		938
939	/* init dig PHYs */	939	/* init dig PHYs */
940	if (rdev->is_atom_bios)	940	if (rdev->is_atom_bios)
941	radeon_atom_encoder_init(rdev);	941	radeon_atom_encoder_init(rdev);
942	/* reset hpd state */	942	/* reset hpd state */
943	radeon_hpd_init(rdev);	943	radeon_hpd_init(rdev);
944	/* blat the mode back in */	944	/* blat the mode back in */
945	drm_helper_resume_force_mode(dev);	945	drm_helper_resume_force_mode(dev);
946	/* turn on display hw */	946	/* turn on display hw */
947	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {	947	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
948	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);	948	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
949	}	949	}
950	return 0;	950	return 0;
951	}	951	}
952		952
953	int radeon_gpu_reset(struct radeon_device *rdev)	953	int radeon_gpu_reset(struct radeon_device *rdev)
954	{	954	{
955	int r;	955	int r;
956	int resched;	956	int resched;
957		957
		958	/* Prevent CS ioctl from interfering */
		959	radeon_mutex_lock(&rdev->cs_mutex);
		960
958	radeon_save_bios_scratch_regs(rdev);	961	radeon_save_bios_scratch_regs(rdev);
959	/* block TTM */	962	/* block TTM */
960	resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);	963	resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);
961	radeon_suspend(rdev);	964	radeon_suspend(rdev);
962		965
963	r = radeon_asic_reset(rdev);	966	r = radeon_asic_reset(rdev);
964	if (!r) {	967	if (!r) {
965	dev_info(rdev->dev, "GPU reset succeed\n");	968	dev_info(rdev->dev, "GPU reset succeed\n");
966	radeon_resume(rdev);	969	radeon_resume(rdev);
967	radeon_restore_bios_scratch_regs(rdev);	970	radeon_restore_bios_scratch_regs(rdev);
968	drm_helper_resume_force_mode(rdev->ddev);	971	drm_helper_resume_force_mode(rdev->ddev);
969	ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);	972	ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
970	return 0;
971	}	973	}
972	/* bad news, how to tell it to userspace ? */	974
973	dev_info(rdev->dev, "GPU reset failed\n");	975	radeon_mutex_unlock(&rdev->cs_mutex);
		976
		977	if (r) {
		978	/* bad news, how to tell it to userspace ? */
		979	dev_info(rdev->dev, "GPU reset failed\n");
		980	}
		981
974	return r;	982	return r;
975	}	983	}
976		984
977		985
978	/*	986	/*
979	* Debugfs	987	* Debugfs
980	*/	988	*/
981	struct radeon_debugfs {	989	struct radeon_debugfs {
982	struct drm_info_list *files;	990	struct drm_info_list *files;
983	unsigned num_files;	991	unsigned num_files;
984	};	992	};
985	static struct radeon_debugfs _radeon_debugfs[RADEON_DEBUGFS_MAX_COMPONENTS];	993	static struct radeon_debugfs _radeon_debugfs[RADEON_DEBUGFS_MAX_COMPONENTS];
986	static unsigned _radeon_debugfs_count = 0;	994	static unsigned _radeon_debugfs_count = 0;
987		995
988	int radeon_debugfs_add_files(struct radeon_device *rdev,	996	int radeon_debugfs_add_files(struct radeon_device *rdev,
989	struct drm_info_list *files,	997	struct drm_info_list *files,
990	unsigned nfiles)	998	unsigned nfiles)
991	{	999	{
992	unsigned i;	1000	unsigned i;
993		1001
994	for (i = 0; i < _radeon_debugfs_count; i++) {	1002	for (i = 0; i < _radeon_debugfs_count; i++) {
995	if (_radeon_debugfs[i].files == files) {	1003	if (_radeon_debugfs[i].files == files) {
996	/* Already registered */	1004	/* Already registered */
997	return 0;	1005	return 0;
998	}	1006	}
999	}	1007	}
1000		1008
1001	i = _radeon_debugfs_count + 1;	1009	i = _radeon_debugfs_count + 1;
1002	if (i > RADEON_DEBUGFS_MAX_COMPONENTS) {	1010	if (i > RADEON_DEBUGFS_MAX_COMPONENTS) {
1003	DRM_ERROR("Reached maximum number of debugfs components.\n");	1011	DRM_ERROR("Reached maximum number of debugfs components.\n");
1004	DRM_ERROR("Report so we increase "	1012	DRM_ERROR("Report so we increase "
1005	"RADEON_DEBUGFS_MAX_COMPONENTS.\n");	1013	"RADEON_DEBUGFS_MAX_COMPONENTS.\n");
1006	return -EINVAL;	1014	return -EINVAL;
1007	}	1015	}
1008	_radeon_debugfs[_radeon_debugfs_count].files = files;	1016	_radeon_debugfs[_radeon_debugfs_count].files = files;
1009	_radeon_debugfs[_radeon_debugfs_count].num_files = nfiles;	1017	_radeon_debugfs[_radeon_debugfs_count].num_files = nfiles;
1010	_radeon_debugfs_count = i;	1018	_radeon_debugfs_count = i;
1011	#if defined(CONFIG_DEBUG_FS)	1019	#if defined(CONFIG_DEBUG_FS)
1012	drm_debugfs_create_files(files, nfiles,	1020	drm_debugfs_create_files(files, nfiles,
1013	rdev->ddev->control->debugfs_root,	1021	rdev->ddev->control->debugfs_root,
1014	rdev->ddev->control);	1022	rdev->ddev->control);
1015	drm_debugfs_create_files(files, nfiles,	1023	drm_debugfs_create_files(files, nfiles,
1016	rdev->ddev->primary->debugfs_root,	1024	rdev->ddev->primary->debugfs_root,
1017	rdev->ddev->primary);	1025	rdev->ddev->primary);
1018	#endif	1026	#endif
1019	return 0;	1027	return 0;
1020	}	1028	}
1021		1029
1022	#if defined(CONFIG_DEBUG_FS)	1030	#if defined(CONFIG_DEBUG_FS)
1023	int radeon_debugfs_init(struct drm_minor *minor)	1031	int radeon_debugfs_init(struct drm_minor *minor)
1024	{	1032	{
1025	return 0;	1033	return 0;
1026	}	1034	}
1027		1035
1028	void radeon_debugfs_cleanup(struct drm_minor *minor)	1036	void radeon_debugfs_cleanup(struct drm_minor *minor)
1029	{	1037	{
1030	unsigned i;	1038	unsigned i;
1031		1039
1032	for (i = 0; i < _radeon_debugfs_count; i++) {	1040	for (i = 0; i < _radeon_debugfs_count; i++) {
1033	drm_debugfs_remove_files(_radeon_debugfs[i].files,	1041	drm_debugfs_remove_files(_radeon_debugfs[i].files,
1034	_radeon_debugfs[i].num_files, minor);	1042	_radeon_debugfs[i].num_files, minor);
1035	}	1043	}
1036	}	1044	}
1037	#endif	1045	#endif