Eric Lee / smarc-ti-linux-kernel

1

/*

1

/*

2

3

* Author: Alex Williamson <alex.williamson@redhat.com>

3

* Author: Alex Williamson <alex.williamson@redhat.com>

4

*

4

*

5

* This program is free software; you can redistribute it and/or modify

5

* This program is free software; you can redistribute it and/or modify

6

* it under the terms of the GNU General Public License version 2 as

6

* it under the terms of the GNU General Public License version 2 as

7

* published by the Free Software Foundation.

7

* published by the Free Software Foundation.

8

*

8

*

9

* Derived from original vfio:

9

* Derived from original vfio:

10

11

* Author: Tom Lyon, pugs@cisco.com

11

* Author: Tom Lyon, pugs@cisco.com

12

*/

12

*/

13

14

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

15

16

#include <linux/device.h>

16

#include <linux/device.h>

17

#include <linux/eventfd.h>

17

#include <linux/eventfd.h>

18

#include <linux/file.h>

18

#include <linux/file.h>

19

#include <linux/interrupt.h>

19

#include <linux/interrupt.h>

20

#include <linux/iommu.h>

20

#include <linux/iommu.h>

21

#include <linux/module.h>

21

#include <linux/module.h>

22

#include <linux/mutex.h>

22

#include <linux/mutex.h>

23

#include <linux/notifier.h>

23

#include <linux/notifier.h>

24

#include <linux/pci.h>

24

#include <linux/pci.h>

25

#include <linux/pm_runtime.h>

25

#include <linux/pm_runtime.h>

26

#include <linux/slab.h>

26

#include <linux/slab.h>

27

#include <linux/types.h>

27

#include <linux/types.h>

28

#include <linux/uaccess.h>

28

#include <linux/uaccess.h>

29

#include <linux/vfio.h>

29

#include <linux/vfio.h>

30

#include <linux/vgaarb.h>

30

#include <linux/vgaarb.h>

31

32

#include "vfio_pci_private.h"

32

#include "vfio_pci_private.h"

33

34

#define DRIVER_VERSION "0.2"

34

#define DRIVER_VERSION "0.2"

35

#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"

35

#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"

36

#define DRIVER_DESC "VFIO PCI - User Level meta-driver"

36

#define DRIVER_DESC "VFIO PCI - User Level meta-driver"

37

38

static char ids[1024] __initdata;

38

static char ids[1024] __initdata;

39

module_param_string(ids, ids, sizeof(ids), 0);

39

module_param_string(ids, ids, sizeof(ids), 0);

40

MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");

40

MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");

41

42

static bool nointxmask;

42

static bool nointxmask;

43

module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);

43

module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);

44

MODULE_PARM_DESC(nointxmask,

44

MODULE_PARM_DESC(nointxmask,

45

"Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");

45

"Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");

46

47

#ifdef CONFIG_VFIO_PCI_VGA

47

#ifdef CONFIG_VFIO_PCI_VGA

48

static bool disable_vga;

48

static bool disable_vga;

49

module_param(disable_vga, bool, S_IRUGO);

49

module_param(disable_vga, bool, S_IRUGO);

50

MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");

50

MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");

51

#endif

51

#endif

52

53

static bool disable_idle_d3;

53

static bool disable_idle_d3;

54

module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);

54

module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);

55

MODULE_PARM_DESC(disable_idle_d3,

55

MODULE_PARM_DESC(disable_idle_d3,

56

"Disable using the PCI D3 low power state for idle, unused devices");

56

"Disable using the PCI D3 low power state for idle, unused devices");

57

58

static DEFINE_MUTEX(driver_lock);

58

static DEFINE_MUTEX(driver_lock);

59

60

static inline bool vfio_vga_disabled(void)

60

static inline bool vfio_vga_disabled(void)

61

{

61

{

62

#ifdef CONFIG_VFIO_PCI_VGA

62

#ifdef CONFIG_VFIO_PCI_VGA

63

return disable_vga;

63

return disable_vga;

64

#else

64

#else

65

return true;

65

return true;

66

#endif

66

#endif

67

}

67

}

68

69

/*

69

/*

70

* Our VGA arbiter participation is limited since we don't know anything

70

* Our VGA arbiter participation is limited since we don't know anything

71

* about the device itself. However, if the device is the only VGA device

71

* about the device itself. However, if the device is the only VGA device

72

* downstream of a bridge and VFIO VGA support is disabled, then we can

72

* downstream of a bridge and VFIO VGA support is disabled, then we can

73

* safely return legacy VGA IO and memory as not decoded since the user

73

* safely return legacy VGA IO and memory as not decoded since the user

74

* has no way to get to it and routing can be disabled externally at the

74

* has no way to get to it and routing can be disabled externally at the

75

* bridge.

75

* bridge.

76

*/

76

*/

77

static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)

77

static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)

78

{

78

{

79

struct vfio_pci_device *vdev = opaque;

79

struct vfio_pci_device *vdev = opaque;

80

struct pci_dev *tmp = NULL, *pdev = vdev->pdev;

80

struct pci_dev *tmp = NULL, *pdev = vdev->pdev;

81

unsigned char max_busnr;

81

unsigned char max_busnr;

82

unsigned int decodes;

82

unsigned int decodes;

83

84

if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))

84

if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))

85

return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |

85

return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |

86

VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;

86

VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;

87

88

max_busnr = pci_bus_max_busnr(pdev->bus);

88

max_busnr = pci_bus_max_busnr(pdev->bus);

89

decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;

89

decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;

90

91

while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {

91

while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {

92

if (tmp == pdev ||

92

if (tmp == pdev ||

93

pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||

93

pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||

94

pci_is_root_bus(tmp->bus))

94

pci_is_root_bus(tmp->bus))

95

continue;

95

continue;

96

97

if (tmp->bus->number >= pdev->bus->number &&

97

if (tmp->bus->number >= pdev->bus->number &&

98

tmp->bus->number <= max_busnr) {

98

tmp->bus->number <= max_busnr) {

99

pci_dev_put(tmp);

99

pci_dev_put(tmp);

100

decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;

100

decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;

101

break;

101

break;

102

}

102

}

103

}

103

}

104

105

return decodes;

105

return decodes;

106

}

106

}

107

108

static inline bool vfio_pci_is_vga(struct pci_dev *pdev)

108

static inline bool vfio_pci_is_vga(struct pci_dev *pdev)

109

{

109

{

110

return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;

110

return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;

111

}

111

}

112

113

static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)

113

static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)

114

{

114

{

115

struct resource *res;

115

struct resource *res;

116

int bar;

116

int bar;

117

struct vfio_pci_dummy_resource *dummy_res;

117

struct vfio_pci_dummy_resource *dummy_res;

118

119

INIT_LIST_HEAD(&vdev->dummy_resources_list);

119

INIT_LIST_HEAD(&vdev->dummy_resources_list);

120

121

for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {

121

for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {

122

res = vdev->pdev->resource + bar;

122

res = vdev->pdev->resource + bar;

123

124

if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))

124

if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))

125

goto no_mmap;

125

goto no_mmap;

126

127

if (!(res->flags & IORESOURCE_MEM))

127

if (!(res->flags & IORESOURCE_MEM))

128

goto no_mmap;

128

goto no_mmap;

129

130

/*

130

/*

131

* The PCI core shouldn't set up a resource with a

131

* The PCI core shouldn't set up a resource with a

132

* type but zero size. But there may be bugs that

132

* type but zero size. But there may be bugs that

133

* cause us to do that.

133

* cause us to do that.

134

*/

134

*/

135

if (!resource_size(res))

135

if (!resource_size(res))

136

goto no_mmap;

136

goto no_mmap;

137

138

if (resource_size(res) >= PAGE_SIZE) {

138

if (resource_size(res) >= PAGE_SIZE) {

139

vdev->bar_mmap_supported[bar] = true;

139

vdev->bar_mmap_supported[bar] = true;

140

continue;

140

continue;

141

}

141

}

142

143

if (!(res->start & ~PAGE_MASK)) {

143

if (!(res->start & ~PAGE_MASK)) {

144

/*

144

/*

145

* Add a dummy resource to reserve the remainder

145

* Add a dummy resource to reserve the remainder

146

* of the exclusive page in case that hot-add

146

* of the exclusive page in case that hot-add

147

* device's bar is assigned into it.

147

* device's bar is assigned into it.

148

*/

148

*/

149

dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);

149

dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);

150

if (dummy_res == NULL)

150

if (dummy_res == NULL)

151

goto no_mmap;

151

goto no_mmap;

152

153

dummy_res->resource.name = "vfio sub-page reserved";

153

dummy_res->resource.name = "vfio sub-page reserved";

154

dummy_res->resource.start = res->end + 1;

154

dummy_res->resource.start = res->end + 1;

155

dummy_res->resource.end = res->start + PAGE_SIZE - 1;

155

dummy_res->resource.end = res->start + PAGE_SIZE - 1;

156

dummy_res->resource.flags = res->flags;

156

dummy_res->resource.flags = res->flags;

157

if (request_resource(res->parent,

157

if (request_resource(res->parent,

158

&dummy_res->resource)) {

158

&dummy_res->resource)) {

159

kfree(dummy_res);

159

kfree(dummy_res);

160

goto no_mmap;

160

goto no_mmap;

161

}

161

}

162

dummy_res->index = bar;

162

dummy_res->index = bar;

163

list_add(&dummy_res->res_next,

163

list_add(&dummy_res->res_next,

164

&vdev->dummy_resources_list);

164

&vdev->dummy_resources_list);

165

vdev->bar_mmap_supported[bar] = true;

165

vdev->bar_mmap_supported[bar] = true;

166

continue;

166

continue;

167

}

167

}

168

/*

168

/*

169

* Here we don't handle the case when the BAR is not page

169

* Here we don't handle the case when the BAR is not page

170

* aligned because we can't expect the BAR will be

170

* aligned because we can't expect the BAR will be

171

* assigned into the same location in a page in guest

171

* assigned into the same location in a page in guest

172

* when we passthrough the BAR. And it's hard to access

172

* when we passthrough the BAR. And it's hard to access

173

* this BAR in userspace because we have no way to get

173

* this BAR in userspace because we have no way to get

174

* the BAR's location in a page.

174

* the BAR's location in a page.

175

*/

175

*/

176

no_mmap:

176

no_mmap:

177

vdev->bar_mmap_supported[bar] = false;

177

vdev->bar_mmap_supported[bar] = false;

178

}

178

}

179

}

179

}

180

181

static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);

181

static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);

182

static void vfio_pci_disable(struct vfio_pci_device *vdev);

182

static void vfio_pci_disable(struct vfio_pci_device *vdev);

183

184

/*

184

/*

185

* INTx masking requires the ability to disable INTx signaling via PCI_COMMAND

185

* INTx masking requires the ability to disable INTx signaling via PCI_COMMAND

186

* _and_ the ability detect when the device is asserting INTx via PCI_STATUS.

186

* _and_ the ability detect when the device is asserting INTx via PCI_STATUS.

187

* If a device implements the former but not the latter we would typically

187

* If a device implements the former but not the latter we would typically

188

* expect broken_intx_masking be set and require an exclusive interrupt.

188

* expect broken_intx_masking be set and require an exclusive interrupt.

189

* However since we do have control of the device's ability to assert INTx,

189

* However since we do have control of the device's ability to assert INTx,

190

* we can instead pretend that the device does not implement INTx, virtualizing

190

* we can instead pretend that the device does not implement INTx, virtualizing

191

* the pin register to report zero and maintaining DisINTx set on the host.

191

* the pin register to report zero and maintaining DisINTx set on the host.

192

*/

192

*/

193

static bool vfio_pci_nointx(struct pci_dev *pdev)

193

static bool vfio_pci_nointx(struct pci_dev *pdev)

194

{

194

{

195

switch (pdev->vendor) {

195

switch (pdev->vendor) {

196

case PCI_VENDOR_ID_INTEL:

196

case PCI_VENDOR_ID_INTEL:

197

switch (pdev->device) {

197

switch (pdev->device) {

198

/* All i40e (XL710/X710) 10/20/40GbE NICs */

198

/* All i40e (XL710/X710) 10/20/40GbE NICs */

199

case 0x1572:

199

case 0x1572:

200

case 0x1574:

200

case 0x1574:

201

case 0x1580 ... 0x1581:

201

case 0x1580 ... 0x1581:

202

case 0x1583 ... 0x1589:

202

case 0x1583 ... 0x1589:

203

case 0x37d0 ... 0x37d2:

203

case 0x37d0 ... 0x37d2:

204

return true;

204

return true;

205

default:

205

default:

206

return false;

206

return false;

207

}

207

}

208

}

208

}

209

210

return false;

210

return false;

211

}

211

}

212

213

static int vfio_pci_enable(struct vfio_pci_device *vdev)

213

static int vfio_pci_enable(struct vfio_pci_device *vdev)

214

{

214

{

215

struct pci_dev *pdev = vdev->pdev;

215

struct pci_dev *pdev = vdev->pdev;

216

int ret;

216

int ret;

217

u16 cmd;

217

u16 cmd;

218

u8 msix_pos;

218

u8 msix_pos;

219

220

pci_set_power_state(pdev, PCI_D0);

220

pci_set_power_state(pdev, PCI_D0);

221

222

/* Don't allow our initial saved state to include busmaster */

222

/* Don't allow our initial saved state to include busmaster */

223

pci_clear_master(pdev);

223

pci_clear_master(pdev);

224

225

ret = pci_enable_device(pdev);

225

ret = pci_enable_device(pdev);

226

if (ret)

226

if (ret)

227

return ret;

227

return ret;

228

229

vdev->reset_works = (pci_reset_function(pdev) == 0);

229

vdev->reset_works = (pci_reset_function(pdev) == 0);

230

pci_save_state(pdev);

230

pci_save_state(pdev);

231

vdev->pci_saved_state = pci_store_saved_state(pdev);

231

vdev->pci_saved_state = pci_store_saved_state(pdev);

232

if (!vdev->pci_saved_state)

232

if (!vdev->pci_saved_state)

233

pr_debug("%s: Couldn't store %s saved state\n",

233

pr_debug("%s: Couldn't store %s saved state\n",

234

__func__, dev_name(&pdev->dev));

234

__func__, dev_name(&pdev->dev));

235

236

if (likely(!nointxmask)) {

236

if (likely(!nointxmask)) {

237

if (vfio_pci_nointx(pdev)) {

237

if (vfio_pci_nointx(pdev)) {

238

dev_info(&pdev->dev, "Masking broken INTx support\n");

238

dev_info(&pdev->dev, "Masking broken INTx support\n");

239

vdev->nointx = true;

239

vdev->nointx = true;

240

pci_intx(pdev, 0);

240

pci_intx(pdev, 0);

241

} else

241

} else

242

vdev->pci_2_3 = pci_intx_mask_supported(pdev);

242

vdev->pci_2_3 = pci_intx_mask_supported(pdev);

243

}

243

}

244

245

pci_read_config_word(pdev, PCI_COMMAND, &cmd);

245

pci_read_config_word(pdev, PCI_COMMAND, &cmd);

246

if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {

246

if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {

247

cmd &= ~PCI_COMMAND_INTX_DISABLE;

247

cmd &= ~PCI_COMMAND_INTX_DISABLE;

248

pci_write_config_word(pdev, PCI_COMMAND, cmd);

248

pci_write_config_word(pdev, PCI_COMMAND, cmd);

249

}

249

}

250

251

ret = vfio_config_init(vdev);

251

ret = vfio_config_init(vdev);

252

if (ret) {

252

if (ret) {

253

kfree(vdev->pci_saved_state);

253

kfree(vdev->pci_saved_state);

254

vdev->pci_saved_state = NULL;

254

vdev->pci_saved_state = NULL;

255

pci_disable_device(pdev);

255

pci_disable_device(pdev);

256

return ret;

256

return ret;

257

}

257

}

258

259

msix_pos = pdev->msix_cap;

259

msix_pos = pdev->msix_cap;

260

if (msix_pos) {

260

if (msix_pos) {

261

u16 flags;

261

u16 flags;

262

u32 table;

262

u32 table;

263

264

pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);

264

pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);

265

pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);

265

pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);

266

267

vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;

267

vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;

268

vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;

268

vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;

269

vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;

269

vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;

270

} else

270

} else

271

vdev->msix_bar = 0xFF;

271

vdev->msix_bar = 0xFF;

272

273

if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))

273

if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))

274

vdev->has_vga = true;

274

vdev->has_vga = true;

275

276

277

if (vfio_pci_is_vga(pdev) &&

277

if (vfio_pci_is_vga(pdev) &&

278

pdev->vendor == PCI_VENDOR_ID_INTEL &&

278

pdev->vendor == PCI_VENDOR_ID_INTEL &&

279

IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {

279

IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {

280

ret = vfio_pci_igd_init(vdev);

280

ret = vfio_pci_igd_init(vdev);

281

if (ret) {

281

if (ret) {

282

dev_warn(&vdev->pdev->dev,

282

dev_warn(&vdev->pdev->dev,

283

"Failed to setup Intel IGD regions\n");

283

"Failed to setup Intel IGD regions\n");

284

vfio_pci_disable(vdev);

284

vfio_pci_disable(vdev);

285

return ret;

285

return ret;

286

}

286

}

287

}

287

}

288

289

vfio_pci_probe_mmaps(vdev);

289

vfio_pci_probe_mmaps(vdev);

290

291

return 0;

291

return 0;

292

}

292

}

293

294

static void vfio_pci_disable(struct vfio_pci_device *vdev)

294

static void vfio_pci_disable(struct vfio_pci_device *vdev)

295

{

295

{

296

struct pci_dev *pdev = vdev->pdev;

296

struct pci_dev *pdev = vdev->pdev;

297

struct vfio_pci_dummy_resource *dummy_res, *tmp;

297

struct vfio_pci_dummy_resource *dummy_res, *tmp;

298

int i, bar;

298

int i, bar;

299

300

/* Stop the device from further DMA */

300

/* Stop the device from further DMA */

301

pci_clear_master(pdev);

301

pci_clear_master(pdev);

302

303

vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |

303

vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |

304

VFIO_IRQ_SET_ACTION_TRIGGER,

304

VFIO_IRQ_SET_ACTION_TRIGGER,

305

vdev->irq_type, 0, 0, NULL);

305

vdev->irq_type, 0, 0, NULL);

306

307

vdev->virq_disabled = false;

307

vdev->virq_disabled = false;

308

309

for (i = 0; i < vdev->num_regions; i++)

309

for (i = 0; i < vdev->num_regions; i++)

310

vdev->region[i].ops->release(vdev, &vdev->region[i]);

310

vdev->region[i].ops->release(vdev, &vdev->region[i]);

311

312

vdev->num_regions = 0;

312

vdev->num_regions = 0;

313

kfree(vdev->region);

313

kfree(vdev->region);

314

vdev->region = NULL; /* don't krealloc a freed pointer */

314

vdev->region = NULL; /* don't krealloc a freed pointer */

315

316

vfio_config_free(vdev);

316

vfio_config_free(vdev);

317

318

for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {

318

for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {

319

if (!vdev->barmap[bar])

319

if (!vdev->barmap[bar])

320

continue;

320

continue;

321

pci_iounmap(pdev, vdev->barmap[bar]);

321

pci_iounmap(pdev, vdev->barmap[bar]);

322

pci_release_selected_regions(pdev, 1 << bar);

322

pci_release_selected_regions(pdev, 1 << bar);

323

vdev->barmap[bar] = NULL;

323

vdev->barmap[bar] = NULL;

324

}

324

}

325

326

list_for_each_entry_safe(dummy_res, tmp,

326

list_for_each_entry_safe(dummy_res, tmp,

327

&vdev->dummy_resources_list, res_next) {

327

&vdev->dummy_resources_list, res_next) {

328

list_del(&dummy_res->res_next);

328

list_del(&dummy_res->res_next);

329

release_resource(&dummy_res->resource);

329

release_resource(&dummy_res->resource);

330

kfree(dummy_res);

330

kfree(dummy_res);

331

}

331

}

332

333

vdev->needs_reset = true;

333

vdev->needs_reset = true;

334

335

/*

335

/*

336

* If we have saved state, restore it. If we can reset the device,

336

* If we have saved state, restore it. If we can reset the device,

337

* even better. Resetting with current state seems better than

337

* even better. Resetting with current state seems better than

338

* nothing, but saving and restoring current state without reset

338

* nothing, but saving and restoring current state without reset

339

* is just busy work.

339

* is just busy work.

340

*/

340

*/

341

if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {

341

if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {

342

pr_info("%s: Couldn't reload %s saved state\n",

342

pr_info("%s: Couldn't reload %s saved state\n",

343

__func__, dev_name(&pdev->dev));

343

__func__, dev_name(&pdev->dev));

344

345

if (!vdev->reset_works)

345

if (!vdev->reset_works)

346

goto out;

346

goto out;

347

348

pci_save_state(pdev);

348

pci_save_state(pdev);

349

}

349

}

350

351

/*

351

/*

352

* Disable INTx and MSI, presumably to avoid spurious interrupts

352

* Disable INTx and MSI, presumably to avoid spurious interrupts

353

* during reset. Stolen from pci_reset_function()

353

* during reset. Stolen from pci_reset_function()

354

*/

354

*/

355

pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);

355

pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);

356

357

/*

357

/*

358

* Try to reset the device. The success of this is dependent on

358

* Try to reset the device. The success of this is dependent on

359

* being able to lock the device, which is not always possible.

359

* being able to lock the device, which is not always possible.

360

*/

360

*/

361

if (vdev->reset_works && !pci_try_reset_function(pdev))

361

if (vdev->reset_works && !pci_try_reset_function(pdev))

362

vdev->needs_reset = false;

362

vdev->needs_reset = false;

363

364

pci_restore_state(pdev);

364

pci_restore_state(pdev);

365

out:

365

out:

366

pci_disable_device(pdev);

366

pci_disable_device(pdev);

367

368

vfio_pci_try_bus_reset(vdev);

368

vfio_pci_try_bus_reset(vdev);

369

370

if (!disable_idle_d3)

370

if (!disable_idle_d3)

371

pci_set_power_state(pdev, PCI_D3hot);

371

pci_set_power_state(pdev, PCI_D3hot);

372

}

372

}

373

374

static void vfio_pci_release(void *device_data)

374

static void vfio_pci_release(void *device_data)

375

{

375

{

376

struct vfio_pci_device *vdev = device_data;

376

struct vfio_pci_device *vdev = device_data;

377

378

mutex_lock(&driver_lock);

378

mutex_lock(&driver_lock);

379

380

if (!(--vdev->refcnt)) {

380

if (!(--vdev->refcnt)) {

381

vfio_spapr_pci_eeh_release(vdev->pdev);

381

vfio_spapr_pci_eeh_release(vdev->pdev);

382

vfio_pci_disable(vdev);

382

vfio_pci_disable(vdev);

383

}

383

}

384

385

mutex_unlock(&driver_lock);

385

mutex_unlock(&driver_lock);

386

387

module_put(THIS_MODULE);

387

module_put(THIS_MODULE);

388

}

388

}

389

390

static int vfio_pci_open(void *device_data)

390

static int vfio_pci_open(void *device_data)

391

{

391

{

392

struct vfio_pci_device *vdev = device_data;

392

struct vfio_pci_device *vdev = device_data;

393

int ret = 0;

393

int ret = 0;

394

395

if (!try_module_get(THIS_MODULE))

395

if (!try_module_get(THIS_MODULE))

396

return -ENODEV;

396

return -ENODEV;

397

398

mutex_lock(&driver_lock);

398

mutex_lock(&driver_lock);

399

400

if (!vdev->refcnt) {

400

if (!vdev->refcnt) {

401

ret = vfio_pci_enable(vdev);

401

ret = vfio_pci_enable(vdev);

402

if (ret)

402

if (ret)

403

goto error;

403

goto error;

404

405

vfio_spapr_pci_eeh_open(vdev->pdev);

405

vfio_spapr_pci_eeh_open(vdev->pdev);

406

}

406

}

407

vdev->refcnt++;

407

vdev->refcnt++;

408

error:

408

error:

409

mutex_unlock(&driver_lock);

409

mutex_unlock(&driver_lock);

410

if (ret)

410

if (ret)

411

module_put(THIS_MODULE);

411

module_put(THIS_MODULE);

412

return ret;

412

return ret;

413

}

413

}

414

415

static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)

415

static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)

416

{

416

{

417

if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {

417

if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {

418

u8 pin;

418

u8 pin;

419

pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);

419

pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);

420

if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin)

420

if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin)

421

return 1;

421

return 1;

422

423

} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {

423

} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {

424

u8 pos;

424

u8 pos;

425

u16 flags;

425

u16 flags;

426

427

pos = vdev->pdev->msi_cap;

427

pos = vdev->pdev->msi_cap;

428

if (pos) {

428

if (pos) {

429

pci_read_config_word(vdev->pdev,

429

pci_read_config_word(vdev->pdev,

430

pos + PCI_MSI_FLAGS, &flags);

430

pos + PCI_MSI_FLAGS, &flags);

431

return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);

431

return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);

432

}

432

}

433

} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {

433

} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {

434

u8 pos;

434

u8 pos;

435

u16 flags;

435

u16 flags;

436

437

pos = vdev->pdev->msix_cap;

437

pos = vdev->pdev->msix_cap;

438

if (pos) {

438

if (pos) {

439

pci_read_config_word(vdev->pdev,

439

pci_read_config_word(vdev->pdev,

440

pos + PCI_MSIX_FLAGS, &flags);

440

pos + PCI_MSIX_FLAGS, &flags);

441

442

return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;

442

return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;

443

}

443

}

444

} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {

444

} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {

445

if (pci_is_pcie(vdev->pdev))

445

if (pci_is_pcie(vdev->pdev))

446

return 1;

446

return 1;

447

} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {

447

} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {

448

return 1;

448

return 1;

449

}

449

}

450

451

return 0;

451

return 0;

452

}

452

}

453

454

static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)

454

static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)

455

{

455

{

456

(*(int *)data)++;

456

(*(int *)data)++;

457

return 0;

457

return 0;

458

}

458

}

459

460

struct vfio_pci_fill_info {

460

struct vfio_pci_fill_info {

461

int max;

461

int max;

462

int cur;

462

int cur;

463

struct vfio_pci_dependent_device *devices;

463

struct vfio_pci_dependent_device *devices;

464

};

464

};

465

466

static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)

466

static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)

467

{

467

{

468

struct vfio_pci_fill_info *fill = data;

468

struct vfio_pci_fill_info *fill = data;

469

struct iommu_group *iommu_group;

469

struct iommu_group *iommu_group;

470

471

if (fill->cur == fill->max)

471

if (fill->cur == fill->max)

472

return -EAGAIN; /* Something changed, try again */

472

return -EAGAIN; /* Something changed, try again */

473

474

iommu_group = iommu_group_get(&pdev->dev);

474

iommu_group = iommu_group_get(&pdev->dev);

475

if (!iommu_group)

475

if (!iommu_group)

476

return -EPERM; /* Cannot reset non-isolated devices */

476

return -EPERM; /* Cannot reset non-isolated devices */

477

478

fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);

478

fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);

479

fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);

479

fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);

480

fill->devices[fill->cur].bus = pdev->bus->number;

480

fill->devices[fill->cur].bus = pdev->bus->number;

481

fill->devices[fill->cur].devfn = pdev->devfn;

481

fill->devices[fill->cur].devfn = pdev->devfn;

482

fill->cur++;

482

fill->cur++;

483

iommu_group_put(iommu_group);

483

iommu_group_put(iommu_group);

484

return 0;

484

return 0;

485

}

485

}

486

487

struct vfio_pci_group_entry {

487

struct vfio_pci_group_entry {

488

struct vfio_group *group;

488

struct vfio_group *group;

489

int id;

489

int id;

490

};

490

};

491

492

struct vfio_pci_group_info {

492

struct vfio_pci_group_info {

493

int count;

493

int count;

494

struct vfio_pci_group_entry *groups;

494

struct vfio_pci_group_entry *groups;

495

};

495

};

496

497

static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)

497

static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)

498

{

498

{

499

struct vfio_pci_group_info *info = data;

499

struct vfio_pci_group_info *info = data;

500

struct iommu_group *group;

500

struct iommu_group *group;

501

int id, i;

501

int id, i;

502

503

group = iommu_group_get(&pdev->dev);

503

group = iommu_group_get(&pdev->dev);

504

if (!group)

504

if (!group)

505

return -EPERM;

505

return -EPERM;

506

507

id = iommu_group_id(group);

507

id = iommu_group_id(group);

508

509

for (i = 0; i < info->count; i++)

509

for (i = 0; i < info->count; i++)

510

if (info->groups[i].id == id)

510

if (info->groups[i].id == id)

511

break;

511

break;

512

513

iommu_group_put(group);

513

iommu_group_put(group);

514

515

return (i == info->count) ? -EINVAL : 0;

515

return (i == info->count) ? -EINVAL : 0;

516

}

516

}

517

518

static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)

518

static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)

519

{

519

{

520

for (; pdev; pdev = pdev->bus->self)

520

for (; pdev; pdev = pdev->bus->self)

521

if (pdev->bus == slot->bus)

521

if (pdev->bus == slot->bus)

522

return (pdev->slot == slot);

522

return (pdev->slot == slot);

523

return false;

523

return false;

524

}

524

}

525

526

struct vfio_pci_walk_info {

526

struct vfio_pci_walk_info {

527

int (*fn)(struct pci_dev *, void *data);

527

int (*fn)(struct pci_dev *, void *data);

528

void *data;

528

void *data;

529

struct pci_dev *pdev;

529

struct pci_dev *pdev;

530

bool slot;

530

bool slot;

531

int ret;

531

int ret;

532

};

532

};

533

534

static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)

534

static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)

535

{

535

{

536

struct vfio_pci_walk_info *walk = data;

536

struct vfio_pci_walk_info *walk = data;

537

538

if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))

538

if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))

539

walk->ret = walk->fn(pdev, walk->data);

539

walk->ret = walk->fn(pdev, walk->data);

540

541

return walk->ret;

541

return walk->ret;

542

}

542

}

543

544

static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,

544

static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,

545

int (*fn)(struct pci_dev *,

545

int (*fn)(struct pci_dev *,

546

void *data), void *data,

546

void *data), void *data,

547

bool slot)

547

bool slot)

548

{

548

{

549

struct vfio_pci_walk_info walk = {

549

struct vfio_pci_walk_info walk = {

550

.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,

550

.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,

551

};

551

};

552

553

pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);

553

pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);

554

555

return walk.ret;

555

return walk.ret;

556

}

556

}

557

558

static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,

558

static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,

559

struct vfio_info_cap *caps)

559

struct vfio_info_cap *caps)

560

{

560

{

561

struct vfio_info_cap_header *header;

561

struct vfio_info_cap_header *header;

562

struct vfio_region_info_cap_sparse_mmap *sparse;

562

struct vfio_region_info_cap_sparse_mmap *sparse;

563

size_t end, size;

563

size_t end, size;

564

int nr_areas = 2, i = 0;

564

int nr_areas = 2, i = 0;

565

566

end = pci_resource_len(vdev->pdev, vdev->msix_bar);

566

end = pci_resource_len(vdev->pdev, vdev->msix_bar);

567

568

/* If MSI-X table is aligned to the start or end, only one area */

568

/* If MSI-X table is aligned to the start or end, only one area */

569

if (((vdev->msix_offset & PAGE_MASK) == 0) ||

569

if (((vdev->msix_offset & PAGE_MASK) == 0) ||

570

(PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) >= end))

570

(PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) >= end))

571

nr_areas = 1;

571

nr_areas = 1;

572

573

size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));

573

size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));

574

575

header = vfio_info_cap_add(caps, size,

575

header = vfio_info_cap_add(caps, size,

576

VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);

576

VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);

577

if (IS_ERR(header))

577

if (IS_ERR(header))

578

return PTR_ERR(header);

578

return PTR_ERR(header);

579

580

sparse = container_of(header,

580

sparse = container_of(header,

581

struct vfio_region_info_cap_sparse_mmap, header);

581

struct vfio_region_info_cap_sparse_mmap, header);

582

sparse->nr_areas = nr_areas;

582

sparse->nr_areas = nr_areas;

583

584

if (vdev->msix_offset & PAGE_MASK) {

584

if (vdev->msix_offset & PAGE_MASK) {

585

sparse->areas[i].offset = 0;

585

sparse->areas[i].offset = 0;

586

sparse->areas[i].size = vdev->msix_offset & PAGE_MASK;

586

sparse->areas[i].size = vdev->msix_offset & PAGE_MASK;

587

i++;

587

i++;

588

}

588

}

589

590

if (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) < end) {

590

if (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) < end) {

591

sparse->areas[i].offset = PAGE_ALIGN(vdev->msix_offset +

591

sparse->areas[i].offset = PAGE_ALIGN(vdev->msix_offset +

592

vdev->msix_size);

592

vdev->msix_size);

593

sparse->areas[i].size = end - sparse->areas[i].offset;

593

sparse->areas[i].size = end - sparse->areas[i].offset;

594

i++;

594

i++;

595

}

595

}

596

597

return 0;

597

return 0;

598

}

598

}

599

600

static int region_type_cap(struct vfio_pci_device *vdev,

600

static int region_type_cap(struct vfio_pci_device *vdev,

601

struct vfio_info_cap *caps,

601

struct vfio_info_cap *caps,

602

unsigned int type, unsigned int subtype)

602

unsigned int type, unsigned int subtype)

603

{

603

{

604

struct vfio_info_cap_header *header;

604

struct vfio_info_cap_header *header;

605

struct vfio_region_info_cap_type *cap;

605

struct vfio_region_info_cap_type *cap;

606

607

header = vfio_info_cap_add(caps, sizeof(*cap),

607

header = vfio_info_cap_add(caps, sizeof(*cap),

608

VFIO_REGION_INFO_CAP_TYPE, 1);

608

VFIO_REGION_INFO_CAP_TYPE, 1);

609

if (IS_ERR(header))

609

if (IS_ERR(header))

610

return PTR_ERR(header);

610

return PTR_ERR(header);

611

612

cap = container_of(header, struct vfio_region_info_cap_type, header);

612

cap = container_of(header, struct vfio_region_info_cap_type, header);

613

cap->type = type;

613

cap->type = type;

614

cap->subtype = subtype;

614

cap->subtype = subtype;

615

616

return 0;

616

return 0;

617

}

617

}

618

619

int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,

619

int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,

620

unsigned int type, unsigned int subtype,

620

unsigned int type, unsigned int subtype,

621

const struct vfio_pci_regops *ops,

621

const struct vfio_pci_regops *ops,

622

size_t size, u32 flags, void *data)

622

size_t size, u32 flags, void *data)

623

{

623

{

624

struct vfio_pci_region *region;

624

struct vfio_pci_region *region;

625

626

region = krealloc(vdev->region,

626

region = krealloc(vdev->region,

627

(vdev->num_regions + 1) * sizeof(*region),

627

(vdev->num_regions + 1) * sizeof(*region),

628

GFP_KERNEL);

628

GFP_KERNEL);

629

if (!region)

629

if (!region)

630

return -ENOMEM;

630

return -ENOMEM;

631

632

vdev->region = region;

632

vdev->region = region;

633

vdev->region[vdev->num_regions].type = type;

633

vdev->region[vdev->num_regions].type = type;

634

vdev->region[vdev->num_regions].subtype = subtype;

634

vdev->region[vdev->num_regions].subtype = subtype;

635

vdev->region[vdev->num_regions].ops = ops;

635

vdev->region[vdev->num_regions].ops = ops;

636

vdev->region[vdev->num_regions].size = size;

636

vdev->region[vdev->num_regions].size = size;

637

vdev->region[vdev->num_regions].flags = flags;

637

vdev->region[vdev->num_regions].flags = flags;

638

vdev->region[vdev->num_regions].data = data;

638

vdev->region[vdev->num_regions].data = data;

639

640

vdev->num_regions++;

640

vdev->num_regions++;

641

642

return 0;

642

return 0;

643

}

643

}

644

645

static long vfio_pci_ioctl(void *device_data,

645

static long vfio_pci_ioctl(void *device_data,

646

unsigned int cmd, unsigned long arg)

646

unsigned int cmd, unsigned long arg)

647

{

647

{

648

struct vfio_pci_device *vdev = device_data;

648

struct vfio_pci_device *vdev = device_data;

649

unsigned long minsz;

649

unsigned long minsz;

650

651

if (cmd == VFIO_DEVICE_GET_INFO) {

651

if (cmd == VFIO_DEVICE_GET_INFO) {

652

struct vfio_device_info info;

652

struct vfio_device_info info;

653

654

minsz = offsetofend(struct vfio_device_info, num_irqs);

654

minsz = offsetofend(struct vfio_device_info, num_irqs);

655

656

if (copy_from_user(&info, (void __user *)arg, minsz))

656

if (copy_from_user(&info, (void __user *)arg, minsz))

657

return -EFAULT;

657

return -EFAULT;

658

659

if (info.argsz < minsz)

659

if (info.argsz < minsz)

660

return -EINVAL;

660

return -EINVAL;

661

662

info.flags = VFIO_DEVICE_FLAGS_PCI;

662

info.flags = VFIO_DEVICE_FLAGS_PCI;

663

664

if (vdev->reset_works)

664

if (vdev->reset_works)

665

info.flags |= VFIO_DEVICE_FLAGS_RESET;

665

info.flags |= VFIO_DEVICE_FLAGS_RESET;

666

667

info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;

667

info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;

668

info.num_irqs = VFIO_PCI_NUM_IRQS;

668

info.num_irqs = VFIO_PCI_NUM_IRQS;

669

670

return copy_to_user((void __user *)arg, &info, minsz) ?

670

return copy_to_user((void __user *)arg, &info, minsz) ?

671

-EFAULT : 0;

671

-EFAULT : 0;

672

673

} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {

673

} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {

674

struct pci_dev *pdev = vdev->pdev;

674

struct pci_dev *pdev = vdev->pdev;

675

struct vfio_region_info info;

675

struct vfio_region_info info;

676

struct vfio_info_cap caps = { .buf = NULL, .size = 0 };

676

struct vfio_info_cap caps = { .buf = NULL, .size = 0 };

677

int i, ret;

677

int i, ret;

678

679

minsz = offsetofend(struct vfio_region_info, offset);

679

minsz = offsetofend(struct vfio_region_info, offset);

680

681

if (copy_from_user(&info, (void __user *)arg, minsz))

681

if (copy_from_user(&info, (void __user *)arg, minsz))

682

return -EFAULT;

682

return -EFAULT;

683

684

if (info.argsz < minsz)

684

if (info.argsz < minsz)

685

return -EINVAL;

685

return -EINVAL;

686

687

switch (info.index) {

687

switch (info.index) {

688

case VFIO_PCI_CONFIG_REGION_INDEX:

688

case VFIO_PCI_CONFIG_REGION_INDEX:

689

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

689

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

690

info.size = pdev->cfg_size;

690

info.size = pdev->cfg_size;

691

info.flags = VFIO_REGION_INFO_FLAG_READ |

691

info.flags = VFIO_REGION_INFO_FLAG_READ |

692

VFIO_REGION_INFO_FLAG_WRITE;

692

VFIO_REGION_INFO_FLAG_WRITE;

693

break;

693

break;

694

case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:

694

case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:

695

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

695

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

696

info.size = pci_resource_len(pdev, info.index);

696

info.size = pci_resource_len(pdev, info.index);

697

if (!info.size) {

697

if (!info.size) {

698

info.flags = 0;

698

info.flags = 0;

699

break;

699

break;

700

}

700

}

701

702

info.flags = VFIO_REGION_INFO_FLAG_READ |

702

info.flags = VFIO_REGION_INFO_FLAG_READ |

703

VFIO_REGION_INFO_FLAG_WRITE;

703

VFIO_REGION_INFO_FLAG_WRITE;

704

if (vdev->bar_mmap_supported[info.index]) {

704

if (vdev->bar_mmap_supported[info.index]) {

705

info.flags |= VFIO_REGION_INFO_FLAG_MMAP;

705

info.flags |= VFIO_REGION_INFO_FLAG_MMAP;

706

if (info.index == vdev->msix_bar) {

706

if (info.index == vdev->msix_bar) {

707

ret = msix_sparse_mmap_cap(vdev, &caps);

707

ret = msix_sparse_mmap_cap(vdev, &caps);

708

if (ret)

708

if (ret)

709

return ret;

709

return ret;

710

}

710

}

711

}

711

}

712

713

break;

713

break;

714

case VFIO_PCI_ROM_REGION_INDEX:

714

case VFIO_PCI_ROM_REGION_INDEX:

715

{

715

{

716

void __iomem *io;

716

void __iomem *io;

717

size_t size;

717

size_t size;

718

719

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

719

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

720

info.flags = 0;

720

info.flags = 0;

721

722

/* Report the BAR size, not the ROM size */

722

/* Report the BAR size, not the ROM size */

723

info.size = pci_resource_len(pdev, info.index);

723

info.size = pci_resource_len(pdev, info.index);

724

if (!info.size) {

724

if (!info.size) {

725

/* Shadow ROMs appear as PCI option ROMs */

725

/* Shadow ROMs appear as PCI option ROMs */

726

if (pdev->resource[PCI_ROM_RESOURCE].flags &

726

if (pdev->resource[PCI_ROM_RESOURCE].flags &

727

IORESOURCE_ROM_SHADOW)

727

IORESOURCE_ROM_SHADOW)

728

info.size = 0x20000;

728

info.size = 0x20000;

729

else

729

else

730

break;

730

break;

731

}

731

}

732

733

/* Is it really there? */

733

/* Is it really there? */

734

io = pci_map_rom(pdev, &size);

734

io = pci_map_rom(pdev, &size);

735

if (!io || !size) {

735

if (!io || !size) {

736

info.size = 0;

736

info.size = 0;

737

break;

737

break;

738

}

738

}

739

pci_unmap_rom(pdev, io);

739

pci_unmap_rom(pdev, io);

740

741

info.flags = VFIO_REGION_INFO_FLAG_READ;

741

info.flags = VFIO_REGION_INFO_FLAG_READ;

742

break;

742

break;

743

}

743

}

744

case VFIO_PCI_VGA_REGION_INDEX:

744

case VFIO_PCI_VGA_REGION_INDEX:

745

if (!vdev->has_vga)

745

if (!vdev->has_vga)

746

return -EINVAL;

746

return -EINVAL;

747

748

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

748

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

749

info.size = 0xc0000;

749

info.size = 0xc0000;

750

info.flags = VFIO_REGION_INFO_FLAG_READ |

750

info.flags = VFIO_REGION_INFO_FLAG_READ |

751

VFIO_REGION_INFO_FLAG_WRITE;

751

VFIO_REGION_INFO_FLAG_WRITE;

752

753

break;

753

break;

754

default:

754

default:

755

if (info.index >=

755

if (info.index >=

756

VFIO_PCI_NUM_REGIONS + vdev->num_regions)

756

VFIO_PCI_NUM_REGIONS + vdev->num_regions)

757

return -EINVAL;

757

return -EINVAL;

758

759

i = info.index - VFIO_PCI_NUM_REGIONS;

759

i = info.index - VFIO_PCI_NUM_REGIONS;

760

761

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

761

info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);

762

info.size = vdev->region[i].size;

762

info.size = vdev->region[i].size;

763

info.flags = vdev->region[i].flags;

763

info.flags = vdev->region[i].flags;

764

765

ret = region_type_cap(vdev, &caps,

765

ret = region_type_cap(vdev, &caps,

766

vdev->region[i].type,

766

vdev->region[i].type,

767

vdev->region[i].subtype);

767

vdev->region[i].subtype);

768

if (ret)

768

if (ret)

769

return ret;

769

return ret;

770

}

770

}

771

772

if (caps.size) {

772

if (caps.size) {

773

info.flags |= VFIO_REGION_INFO_FLAG_CAPS;

773

info.flags |= VFIO_REGION_INFO_FLAG_CAPS;

774

if (info.argsz < sizeof(info) + caps.size) {

774

if (info.argsz < sizeof(info) + caps.size) {

775

info.argsz = sizeof(info) + caps.size;

775

info.argsz = sizeof(info) + caps.size;

776

info.cap_offset = 0;

776

info.cap_offset = 0;

777

} else {

777

} else {

778

vfio_info_cap_shift(&caps, sizeof(info));

778

vfio_info_cap_shift(&caps, sizeof(info));

779

if (copy_to_user((void __user *)arg +

779

if (copy_to_user((void __user *)arg +

780

sizeof(info), caps.buf,

780

sizeof(info), caps.buf,

781

caps.size)) {

781

caps.size)) {

782

kfree(caps.buf);

782

kfree(caps.buf);

783

return -EFAULT;

783

return -EFAULT;

784

}

784

}

785

info.cap_offset = sizeof(info);

785

info.cap_offset = sizeof(info);

786

}

786

}

787

788

kfree(caps.buf);

788

kfree(caps.buf);

789

}

789

}

790

791

return copy_to_user((void __user *)arg, &info, minsz) ?

791

return copy_to_user((void __user *)arg, &info, minsz) ?

792

-EFAULT : 0;

792

-EFAULT : 0;

793

794

} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {

794

} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {

795

struct vfio_irq_info info;

795

struct vfio_irq_info info;

796

797

minsz = offsetofend(struct vfio_irq_info, count);

797

minsz = offsetofend(struct vfio_irq_info, count);

798

799

if (copy_from_user(&info, (void __user *)arg, minsz))

799

if (copy_from_user(&info, (void __user *)arg, minsz))

800

return -EFAULT;

800

return -EFAULT;

801

802

if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)

802

if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)

803

return -EINVAL;

803

return -EINVAL;

804

805

switch (info.index) {

805

switch (info.index) {

806

case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:

806

case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:

807

case VFIO_PCI_REQ_IRQ_INDEX:

807

case VFIO_PCI_REQ_IRQ_INDEX:

808

break;

808

break;

809

case VFIO_PCI_ERR_IRQ_INDEX:

809

case VFIO_PCI_ERR_IRQ_INDEX:

810

if (pci_is_pcie(vdev->pdev))

810

if (pci_is_pcie(vdev->pdev))

811

break;

811

break;

812

/* pass thru to return error */

812

/* pass thru to return error */

813

default:

813

default:

814

return -EINVAL;

814

return -EINVAL;

815

}

815

}

816

817

info.flags = VFIO_IRQ_INFO_EVENTFD;

817

info.flags = VFIO_IRQ_INFO_EVENTFD;

818

819

info.count = vfio_pci_get_irq_count(vdev, info.index);

819

info.count = vfio_pci_get_irq_count(vdev, info.index);

820

821

if (info.index == VFIO_PCI_INTX_IRQ_INDEX)

821

if (info.index == VFIO_PCI_INTX_IRQ_INDEX)

822

info.flags |= (VFIO_IRQ_INFO_MASKABLE |

822

info.flags |= (VFIO_IRQ_INFO_MASKABLE |

823

VFIO_IRQ_INFO_AUTOMASKED);

823

VFIO_IRQ_INFO_AUTOMASKED);

824

else

824

else

825

info.flags |= VFIO_IRQ_INFO_NORESIZE;

825

info.flags |= VFIO_IRQ_INFO_NORESIZE;

826

827

return copy_to_user((void __user *)arg, &info, minsz) ?

827

return copy_to_user((void __user *)arg, &info, minsz) ?

828

-EFAULT : 0;

828

-EFAULT : 0;

829

830

} else if (cmd == VFIO_DEVICE_SET_IRQS) {

830

} else if (cmd == VFIO_DEVICE_SET_IRQS) {

831

struct vfio_irq_set hdr;

831

struct vfio_irq_set hdr;

832

size_t size;

832

size_t size;

833

u8 *data = NULL;

833

u8 *data = NULL;

834

int max, ret = 0;

834

int max, ret = 0;

835

836

minsz = offsetofend(struct vfio_irq_set, count);

836

minsz = offsetofend(struct vfio_irq_set, count);

837

838

if (copy_from_user(&hdr, (void __user *)arg, minsz))

838

if (copy_from_user(&hdr, (void __user *)arg, minsz))

839

return -EFAULT;

839

return -EFAULT;

840

841

if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||

841

if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||

842

hdr.count >= (U32_MAX - hdr.start) ||

842

hdr.count >= (U32_MAX - hdr.start) ||

843

hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |

843

hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |

844

VFIO_IRQ_SET_ACTION_TYPE_MASK))

844

VFIO_IRQ_SET_ACTION_TYPE_MASK))

845

return -EINVAL;

845

return -EINVAL;

846

847

max = vfio_pci_get_irq_count(vdev, hdr.index);

847

max = vfio_pci_get_irq_count(vdev, hdr.index);

848

if (hdr.start >= max || hdr.start + hdr.count > max)

848

if (hdr.start >= max || hdr.start + hdr.count > max)

849

return -EINVAL;

849

return -EINVAL;

850

851

switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {

851

switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {

852

case VFIO_IRQ_SET_DATA_NONE:

852

case VFIO_IRQ_SET_DATA_NONE:

853

size = 0;

853

size = 0;

854

break;

854

break;

855

case VFIO_IRQ_SET_DATA_BOOL:

855

case VFIO_IRQ_SET_DATA_BOOL:

856

size = sizeof(uint8_t);

856

size = sizeof(uint8_t);

857

break;

857

break;

858

case VFIO_IRQ_SET_DATA_EVENTFD:

858

case VFIO_IRQ_SET_DATA_EVENTFD:

859

size = sizeof(int32_t);

859

size = sizeof(int32_t);

860

break;

860

break;

861

default:

861

default:

862

return -EINVAL;

862

return -EINVAL;

863

}

863

}

864

865

if (size) {

865

if (size) {

866

if (hdr.argsz - minsz < hdr.count * size)

866

if (hdr.argsz - minsz < hdr.count * size)

867

return -EINVAL;

867

return -EINVAL;

868

869

data = memdup_user((void __user *)(arg + minsz),

869

data = memdup_user((void __user *)(arg + minsz),

870

hdr.count * size);

870

hdr.count * size);

871

if (IS_ERR(data))

871

if (IS_ERR(data))

872

return PTR_ERR(data);

872

return PTR_ERR(data);

873

}

873

}

874

875

mutex_lock(&vdev->igate);

875

mutex_lock(&vdev->igate);

876

877

ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,

877

ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,

878

hdr.start, hdr.count, data);

878

hdr.start, hdr.count, data);

879

880

mutex_unlock(&vdev->igate);

880

mutex_unlock(&vdev->igate);

881

kfree(data);

881

kfree(data);

882

883

return ret;

883

return ret;

884

885

} else if (cmd == VFIO_DEVICE_RESET) {

885

} else if (cmd == VFIO_DEVICE_RESET) {

886

return vdev->reset_works ?

886

return vdev->reset_works ?

887

pci_try_reset_function(vdev->pdev) : -EINVAL;

887

pci_try_reset_function(vdev->pdev) : -EINVAL;

888

889

} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {

889

} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {

890

struct vfio_pci_hot_reset_info hdr;

890

struct vfio_pci_hot_reset_info hdr;

891

struct vfio_pci_fill_info fill = { 0 };

891

struct vfio_pci_fill_info fill = { 0 };

892

struct vfio_pci_dependent_device *devices = NULL;

892

struct vfio_pci_dependent_device *devices = NULL;

893

bool slot = false;

893

bool slot = false;

894

int ret = 0;

894

int ret = 0;

895

896

minsz = offsetofend(struct vfio_pci_hot_reset_info, count);

896

minsz = offsetofend(struct vfio_pci_hot_reset_info, count);

897

898

if (copy_from_user(&hdr, (void __user *)arg, minsz))

898

if (copy_from_user(&hdr, (void __user *)arg, minsz))

899

return -EFAULT;

899

return -EFAULT;

900

901

if (hdr.argsz < minsz)

901

if (hdr.argsz < minsz)

902

return -EINVAL;

902

return -EINVAL;

903

904

hdr.flags = 0;

904

hdr.flags = 0;

905

906

/* Can we do a slot or bus reset or neither? */

906

/* Can we do a slot or bus reset or neither? */

907

if (!pci_probe_reset_slot(vdev->pdev->slot))

907

if (!pci_probe_reset_slot(vdev->pdev->slot))

908

slot = true;

908

slot = true;

909

else if (pci_probe_reset_bus(vdev->pdev->bus))

909

else if (pci_probe_reset_bus(vdev->pdev->bus))

910

return -ENODEV;

910

return -ENODEV;

911

912

/* How many devices are affected? */

912

/* How many devices are affected? */

913

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

913

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

914

vfio_pci_count_devs,

914

vfio_pci_count_devs,

915

&fill.max, slot);

915

&fill.max, slot);

916

if (ret)

916

if (ret)

917

return ret;

917

return ret;

918

919

WARN_ON(!fill.max); /* Should always be at least one */

919

WARN_ON(!fill.max); /* Should always be at least one */

920

921

/*

921

/*

922

* If there's enough space, fill it now, otherwise return

922

* If there's enough space, fill it now, otherwise return

923

* -ENOSPC and the number of devices affected.

923

* -ENOSPC and the number of devices affected.

924

*/

924

*/

925

if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {

925

if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {

926

ret = -ENOSPC;

926

ret = -ENOSPC;

927

hdr.count = fill.max;

927

hdr.count = fill.max;

928

goto reset_info_exit;

928

goto reset_info_exit;

929

}

929

}

930

931

devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);

931

devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);

932

if (!devices)

932

if (!devices)

933

return -ENOMEM;

933

return -ENOMEM;

934

935

fill.devices = devices;

935

fill.devices = devices;

936

937

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

937

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

938

vfio_pci_fill_devs,

938

vfio_pci_fill_devs,

939

&fill, slot);

939

&fill, slot);

940

941

/*

941

/*

942

* If a device was removed between counting and filling,

942

* If a device was removed between counting and filling,

943

* we may come up short of fill.max. If a device was

943

* we may come up short of fill.max. If a device was

944

* added, we'll have a return of -EAGAIN above.

944

* added, we'll have a return of -EAGAIN above.

945

*/

945

*/

946

if (!ret)

946

if (!ret)

947

hdr.count = fill.cur;

947

hdr.count = fill.cur;

948

949

reset_info_exit:

949

reset_info_exit:

950

if (copy_to_user((void __user *)arg, &hdr, minsz))

950

if (copy_to_user((void __user *)arg, &hdr, minsz))

951

ret = -EFAULT;

951

ret = -EFAULT;

952

953

if (!ret) {

953

if (!ret) {

954

if (copy_to_user((void __user *)(arg + minsz), devices,

954

if (copy_to_user((void __user *)(arg + minsz), devices,

955

hdr.count * sizeof(*devices)))

955

hdr.count * sizeof(*devices)))

956

ret = -EFAULT;

956

ret = -EFAULT;

957

}

957

}

958

959

kfree(devices);

959

kfree(devices);

960

return ret;

960

return ret;

961

962

} else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {

962

} else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {

963

struct vfio_pci_hot_reset hdr;

963

struct vfio_pci_hot_reset hdr;

964

int32_t *group_fds;

964

int32_t *group_fds;

965

struct vfio_pci_group_entry *groups;

965

struct vfio_pci_group_entry *groups;

966

struct vfio_pci_group_info info;

966

struct vfio_pci_group_info info;

967

bool slot = false;

967

bool slot = false;

968

int i, count = 0, ret = 0;

968

int i, count = 0, ret = 0;

969

970

minsz = offsetofend(struct vfio_pci_hot_reset, count);

970

minsz = offsetofend(struct vfio_pci_hot_reset, count);

971

972

if (copy_from_user(&hdr, (void __user *)arg, minsz))

972

if (copy_from_user(&hdr, (void __user *)arg, minsz))

973

return -EFAULT;

973

return -EFAULT;

974

975

if (hdr.argsz < minsz || hdr.flags)

975

if (hdr.argsz < minsz || hdr.flags)

976

return -EINVAL;

976

return -EINVAL;

977

978

/* Can we do a slot or bus reset or neither? */

978

/* Can we do a slot or bus reset or neither? */

979

if (!pci_probe_reset_slot(vdev->pdev->slot))

979

if (!pci_probe_reset_slot(vdev->pdev->slot))

980

slot = true;

980

slot = true;

981

else if (pci_probe_reset_bus(vdev->pdev->bus))

981

else if (pci_probe_reset_bus(vdev->pdev->bus))

982

return -ENODEV;

982

return -ENODEV;

983

984

/*

984

/*

985

* We can't let userspace give us an arbitrarily large

985

* We can't let userspace give us an arbitrarily large

986

* buffer to copy, so verify how many we think there

986

* buffer to copy, so verify how many we think there

987

* could be. Note groups can have multiple devices so

987

* could be. Note groups can have multiple devices so

988

* one group per device is the max.

988

* one group per device is the max.

989

*/

989

*/

990

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

990

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

991

vfio_pci_count_devs,

991

vfio_pci_count_devs,

992

&count, slot);

992

&count, slot);

993

if (ret)

993

if (ret)

994

return ret;

994

return ret;

995

996

/* Somewhere between 1 and count is OK */

996

/* Somewhere between 1 and count is OK */

997

if (!hdr.count || hdr.count > count)

997

if (!hdr.count || hdr.count > count)

998

return -EINVAL;

998

return -EINVAL;

999

1000

group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);

1000

group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);

1001

groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);

1001

groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);

1002

if (!group_fds || !groups) {

1002

if (!group_fds || !groups) {

1003

kfree(group_fds);

1003

kfree(group_fds);

1004

kfree(groups);

1004

kfree(groups);

1005

return -ENOMEM;

1005

return -ENOMEM;

1006

}

1006

}

1007

1008

if (copy_from_user(group_fds, (void __user *)(arg + minsz),

1008

if (copy_from_user(group_fds, (void __user *)(arg + minsz),

1009

hdr.count * sizeof(*group_fds))) {

1009

hdr.count * sizeof(*group_fds))) {

1010

kfree(group_fds);

1010

kfree(group_fds);

1011

kfree(groups);

1011

kfree(groups);

1012

return -EFAULT;

1012

return -EFAULT;

1013

}

1013

}

1014

1015

/*

1015

/*

1016

* For each group_fd, get the group through the vfio external

1016

* For each group_fd, get the group through the vfio external

1017

* user interface and store the group and iommu ID. This

1017

* user interface and store the group and iommu ID. This

1018

* ensures the group is held across the reset.

1018

* ensures the group is held across the reset.

1019

*/

1019

*/

1020

for (i = 0; i < hdr.count; i++) {

1020

for (i = 0; i < hdr.count; i++) {

1021

struct vfio_group *group;

1021

struct vfio_group *group;

1022

struct fd f = fdget(group_fds[i]);

1022

struct fd f = fdget(group_fds[i]);

1023

if (!f.file) {

1023

if (!f.file) {

1024

ret = -EBADF;

1024

ret = -EBADF;

1025

break;

1025

break;

1026

}

1026

}

1027

1028

group = vfio_group_get_external_user(f.file);

1028

group = vfio_group_get_external_user(f.file);

1029

fdput(f);

1029

fdput(f);

1030

if (IS_ERR(group)) {

1030

if (IS_ERR(group)) {

1031

ret = PTR_ERR(group);

1031

ret = PTR_ERR(group);

1032

break;

1032

break;

1033

}

1033

}

1034

1035

groups[i].group = group;

1035

groups[i].group = group;

1036

groups[i].id = vfio_external_user_iommu_id(group);

1036

groups[i].id = vfio_external_user_iommu_id(group);

1037

}

1037

}

1038

1039

kfree(group_fds);

1039

kfree(group_fds);

1040

1041

/* release reference to groups on error */

1041

/* release reference to groups on error */

1042

if (ret)

1042

if (ret)

1043

goto hot_reset_release;

1043

goto hot_reset_release;

1044

1045

info.count = hdr.count;

1045

info.count = hdr.count;

1046

info.groups = groups;

1046

info.groups = groups;

1047

1048

/*

1048

/*

1049

* Test whether all the affected devices are contained

1049

* Test whether all the affected devices are contained

1050

* by the set of groups provided by the user.

1050

* by the set of groups provided by the user.

1051

*/

1051

*/

1052

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

1052

ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,

1053

vfio_pci_validate_devs,

1053

vfio_pci_validate_devs,

1054

&info, slot);

1054

&info, slot);

1055

if (!ret)

1055

if (!ret)

1056

/* User has access, do the reset */

1056

/* User has access, do the reset */

1057

ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :

1057

ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :

1058

pci_try_reset_bus(vdev->pdev->bus);

1058

pci_try_reset_bus(vdev->pdev->bus);

1059

1060

hot_reset_release:

1060

hot_reset_release:

1061

for (i--; i >= 0; i--)

1061

for (i--; i >= 0; i--)

1062

vfio_group_put_external_user(groups[i].group);

1062

vfio_group_put_external_user(groups[i].group);

1063

1064

kfree(groups);

1064

kfree(groups);

1065

return ret;

1065

return ret;

1066

}

1066

}

1067

1068

return -ENOTTY;

1068

return -ENOTTY;

1069

}

1069

}

1070

1071

static ssize_t vfio_pci_rw(void *device_data, char __user *buf,

1071

static ssize_t vfio_pci_rw(void *device_data, char __user *buf,

1072

size_t count, loff_t *ppos, bool iswrite)

1072

size_t count, loff_t *ppos, bool iswrite)

1073

{

1073

{

1074

unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);

1074

unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);

1075

struct vfio_pci_device *vdev = device_data;

1075

struct vfio_pci_device *vdev = device_data;

1076

1077

if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)

1077

if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)

1078

return -EINVAL;

1078

return -EINVAL;

1079

1080

switch (index) {

1080

switch (index) {

1081

case VFIO_PCI_CONFIG_REGION_INDEX:

1081

case VFIO_PCI_CONFIG_REGION_INDEX:

1082

return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);

1082

return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);

1083

1084

case VFIO_PCI_ROM_REGION_INDEX:

1084

case VFIO_PCI_ROM_REGION_INDEX:

1085

if (iswrite)

1085

if (iswrite)

1086

return -EINVAL;

1086

return -EINVAL;

1087

return vfio_pci_bar_rw(vdev, buf, count, ppos, false);

1087

return vfio_pci_bar_rw(vdev, buf, count, ppos, false);

1088

1089

case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:

1089

case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:

1090

return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);

1090

return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);

1091

1092

case VFIO_PCI_VGA_REGION_INDEX:

1092

case VFIO_PCI_VGA_REGION_INDEX:

1093

return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);

1093

return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);

1094

default:

1094

default:

1095

index -= VFIO_PCI_NUM_REGIONS;

1095

index -= VFIO_PCI_NUM_REGIONS;

1096

return vdev->region[index].ops->rw(vdev, buf,

1096

return vdev->region[index].ops->rw(vdev, buf,

1097

count, ppos, iswrite);

1097

count, ppos, iswrite);

1098

}

1098

}

1099

1100

return -EINVAL;

1100

return -EINVAL;

1101

}

1101

}

1102

1103

static ssize_t vfio_pci_read(void *device_data, char __user *buf,

1103

static ssize_t vfio_pci_read(void *device_data, char __user *buf,

1104

size_t count, loff_t *ppos)

1104

size_t count, loff_t *ppos)

1105

{

1105

{

1106

if (!count)

1106

if (!count)

1107

return 0;

1107

return 0;

1108

1109

return vfio_pci_rw(device_data, buf, count, ppos, false);

1109

return vfio_pci_rw(device_data, buf, count, ppos, false);

1110

}

1110

}

1111

1112

static ssize_t vfio_pci_write(void *device_data, const char __user *buf,

1112

static ssize_t vfio_pci_write(void *device_data, const char __user *buf,

1113

size_t count, loff_t *ppos)

1113

size_t count, loff_t *ppos)

1114

{

1114

{

1115

if (!count)

1115

if (!count)

1116

return 0;

1116

return 0;

1117

1118

return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);

1118

return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);

1119

}

1119

}

1120

1121

static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)

1121

static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)

1122

{

1122

{

1123

struct vfio_pci_device *vdev = device_data;

1123

struct vfio_pci_device *vdev = device_data;

1124

struct pci_dev *pdev = vdev->pdev;

1124

struct pci_dev *pdev = vdev->pdev;

1125

unsigned int index;

1125

unsigned int index;

1126

u64 phys_len, req_len, pgoff, req_start;

1126

u64 phys_len, req_len, pgoff, req_start;

1127

int ret;

1127

int ret;

1128

1129

index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);

1129

index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);

1130

1131

if (vma->vm_end < vma->vm_start)

1131

if (vma->vm_end < vma->vm_start)

1132

return -EINVAL;

1132

return -EINVAL;

1133

if ((vma->vm_flags & VM_SHARED) == 0)

1133

if ((vma->vm_flags & VM_SHARED) == 0)

1134

return -EINVAL;

1134

return -EINVAL;

1135

if (index >= VFIO_PCI_ROM_REGION_INDEX)

1135

if (index >= VFIO_PCI_ROM_REGION_INDEX)

1136

return -EINVAL;

1136

return -EINVAL;

1137

if (!vdev->bar_mmap_supported[index])

1137

if (!vdev->bar_mmap_supported[index])

1138

return -EINVAL;

1138

return -EINVAL;

1139

1140

phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));

1140

phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));

1141

req_len = vma->vm_end - vma->vm_start;

1141

req_len = vma->vm_end - vma->vm_start;

1142

pgoff = vma->vm_pgoff &

1142

pgoff = vma->vm_pgoff &

1143

((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);

1143

((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);

1144

req_start = pgoff << PAGE_SHIFT;

1144

req_start = pgoff << PAGE_SHIFT;

1145

1146

if (req_start + req_len > phys_len)

1146

if (req_start + req_len > phys_len)

1147

return -EINVAL;

1147

return -EINVAL;

1148

1149

if (index == vdev->msix_bar) {

1149

if (index == vdev->msix_bar) {

1150

/*

1150

/*

1151

* Disallow mmaps overlapping the MSI-X table; users don't

1151

* Disallow mmaps overlapping the MSI-X table; users don't

1152

* get to touch this directly. We could find somewhere

1152

* get to touch this directly. We could find somewhere

1153

* else to map the overlap, but page granularity is only

1153

* else to map the overlap, but page granularity is only

1154

* a recommendation, not a requirement, so the user needs

1154

* a recommendation, not a requirement, so the user needs

1155

* to know which bits are real. Requiring them to mmap

1155

* to know which bits are real. Requiring them to mmap

1156

* around the table makes that clear.

1156

* around the table makes that clear.

1157

*/

1157

*/

1158

1159

/* If neither entirely above nor below, then it overlaps */

1159

/* If neither entirely above nor below, then it overlaps */

1160

if (!(req_start >= vdev->msix_offset + vdev->msix_size ||

1160

if (!(req_start >= vdev->msix_offset + vdev->msix_size ||

1161

req_start + req_len <= vdev->msix_offset))

1161

req_start + req_len <= vdev->msix_offset))

1162

return -EINVAL;

1162

return -EINVAL;

1163

}

1163

}

1164

1165

/*

1165

/*

1166

* Even though we don't make use of the barmap for the mmap,

1166

* Even though we don't make use of the barmap for the mmap,

1167

* we need to request the region and the barmap tracks that.

1167

* we need to request the region and the barmap tracks that.

1168

*/

1168

*/

1169

if (!vdev->barmap[index]) {

1169

if (!vdev->barmap[index]) {

1170

ret = pci_request_selected_regions(pdev,

1170

ret = pci_request_selected_regions(pdev,

1171

1 << index, "vfio-pci");

1171

1 << index, "vfio-pci");

1172

if (ret)

1172

if (ret)

1173

return ret;

1173

return ret;

1174

1175

vdev->barmap[index] = pci_iomap(pdev, index, 0);

1175

vdev->barmap[index] = pci_iomap(pdev, index, 0);

1176

if (!vdev->barmap[index]) {

1177

pci_release_selected_regions(pdev, 1 << index);

1178

return -ENOMEM;

1179

}

1176

}

1180

}

1177

1181

1178

vma->vm_private_data = vdev;

1182

vma->vm_private_data = vdev;

1179

vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

1183

vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

1180

vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;

1184

vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;

1181

1185

1182

return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,

1186

return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,

1183

req_len, vma->vm_page_prot);

1187

req_len, vma->vm_page_prot);

1184

}

1188

}

1185

1189

1186

static void vfio_pci_request(void *device_data, unsigned int count)

1190

static void vfio_pci_request(void *device_data, unsigned int count)

1187

{

1191

{

1188

struct vfio_pci_device *vdev = device_data;

1192

struct vfio_pci_device *vdev = device_data;

1189

1193

1190

mutex_lock(&vdev->igate);

1194

mutex_lock(&vdev->igate);

1191

1195

1192

if (vdev->req_trigger) {

1196

if (vdev->req_trigger) {

1193

if (!(count % 10))

1197

if (!(count % 10))

1194

dev_notice_ratelimited(&vdev->pdev->dev,

1198

dev_notice_ratelimited(&vdev->pdev->dev,

1195

"Relaying device request to user (#%u)\n",

1199

"Relaying device request to user (#%u)\n",

1196

count);

1200

count);

1197

eventfd_signal(vdev->req_trigger, 1);

1201

eventfd_signal(vdev->req_trigger, 1);

1198

} else if (count == 0) {

1202

} else if (count == 0) {

1199

dev_warn(&vdev->pdev->dev,

1203

dev_warn(&vdev->pdev->dev,

1200

"No device request channel registered, blocked until released by user\n");

1204

"No device request channel registered, blocked until released by user\n");

1201

}

1205

}

1202

1206

1203

mutex_unlock(&vdev->igate);

1207

mutex_unlock(&vdev->igate);

1204

}

1208

}

1205

1209

1206

static const struct vfio_device_ops vfio_pci_ops = {

1210

static const struct vfio_device_ops vfio_pci_ops = {

1207

.name = "vfio-pci",

1211

.name = "vfio-pci",

1208

.open = vfio_pci_open,

1212

.open = vfio_pci_open,

1209

.release = vfio_pci_release,

1213

.release = vfio_pci_release,

1210

.ioctl = vfio_pci_ioctl,

1214

.ioctl = vfio_pci_ioctl,

1211

.read = vfio_pci_read,

1215

.read = vfio_pci_read,

1212

.write = vfio_pci_write,

1216

.write = vfio_pci_write,

1213

.mmap = vfio_pci_mmap,

1217

.mmap = vfio_pci_mmap,

1214

.request = vfio_pci_request,

1218

.request = vfio_pci_request,

1215

};

1219

};

1216

1220

1217

static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

1221

static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

1218

{

1222

{

1219

struct vfio_pci_device *vdev;

1223

struct vfio_pci_device *vdev;

1220

struct iommu_group *group;

1224

struct iommu_group *group;

1221

int ret;

1225

int ret;

1222

1226

1223

if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)

1227

if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)

1224

return -EINVAL;

1228

return -EINVAL;

1225

1229

1226

group = vfio_iommu_group_get(&pdev->dev);

1230

group = vfio_iommu_group_get(&pdev->dev);

1227

if (!group)

1231

if (!group)

1228

return -EINVAL;

1232

return -EINVAL;

1229

1233

1230

vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);

1234

vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);

1231

if (!vdev) {

1235

if (!vdev) {

1232

vfio_iommu_group_put(group, &pdev->dev);

1236

vfio_iommu_group_put(group, &pdev->dev);

1233

return -ENOMEM;

1237

return -ENOMEM;

1234

}

1238

}

1235

1239

1236

vdev->pdev = pdev;

1240

vdev->pdev = pdev;

1237

vdev->irq_type = VFIO_PCI_NUM_IRQS;

1241

vdev->irq_type = VFIO_PCI_NUM_IRQS;

1238

mutex_init(&vdev->igate);

1242

mutex_init(&vdev->igate);

1239

spin_lock_init(&vdev->irqlock);

1243

spin_lock_init(&vdev->irqlock);

1240

1244

1241

ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);

1245

ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);

1242

if (ret) {

1246

if (ret) {

1243

vfio_iommu_group_put(group, &pdev->dev);

1247

vfio_iommu_group_put(group, &pdev->dev);

1244

kfree(vdev);

1248

kfree(vdev);

1245

return ret;

1249

return ret;

1246

}

1250

}

1247

1251

1248

if (vfio_pci_is_vga(pdev)) {

1252

if (vfio_pci_is_vga(pdev)) {

1249

vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);

1253

vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);

1250

vga_set_legacy_decoding(pdev,

1254

vga_set_legacy_decoding(pdev,

1251

vfio_pci_set_vga_decode(vdev, false));

1255

vfio_pci_set_vga_decode(vdev, false));

1252

}

1256

}

1253

1257

1254

if (!disable_idle_d3) {

1258

if (!disable_idle_d3) {

1255

/*

1259

/*

1256

* pci-core sets the device power state to an unknown value at

1260

* pci-core sets the device power state to an unknown value at

1257

* bootup and after being removed from a driver. The only

1261

* bootup and after being removed from a driver. The only

1258

* transition it allows from this unknown state is to D0, which

1262

* transition it allows from this unknown state is to D0, which

1259

* typically happens when a driver calls pci_enable_device().

1263

* typically happens when a driver calls pci_enable_device().

1260

* We're not ready to enable the device yet, but we do want to

1264

* We're not ready to enable the device yet, but we do want to

1261

* be able to get to D3. Therefore first do a D0 transition

1265

* be able to get to D3. Therefore first do a D0 transition

1262

* before going to D3.

1266

* before going to D3.

1263

*/

1267

*/

1264

pci_set_power_state(pdev, PCI_D0);

1268

pci_set_power_state(pdev, PCI_D0);

1265

pci_set_power_state(pdev, PCI_D3hot);

1269

pci_set_power_state(pdev, PCI_D3hot);

1266

}

1270

}

1267

1271

1268

return ret;

1272

return ret;

1269

}

1273

}

1270

1274

1271

static void vfio_pci_remove(struct pci_dev *pdev)

1275

static void vfio_pci_remove(struct pci_dev *pdev)

1272

{

1276

{

1273

struct vfio_pci_device *vdev;

1277

struct vfio_pci_device *vdev;

1274

1278

1275

vdev = vfio_del_group_dev(&pdev->dev);

1279

vdev = vfio_del_group_dev(&pdev->dev);

1276

if (!vdev)

1280

if (!vdev)

1277

return;

1281

return;

1278

1282

1279

vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);

1283

vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);

1280

kfree(vdev->region);

1284

kfree(vdev->region);

1281

kfree(vdev);

1285

kfree(vdev);

1282

1286

1283

if (vfio_pci_is_vga(pdev)) {

1287

if (vfio_pci_is_vga(pdev)) {

1284

vga_client_register(pdev, NULL, NULL, NULL);

1288

vga_client_register(pdev, NULL, NULL, NULL);

1285

vga_set_legacy_decoding(pdev,

1289

vga_set_legacy_decoding(pdev,

1286

VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |

1290

VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |

1287

VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);

1291

VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);

1288

}

1292

}

1289

1293

1290

if (!disable_idle_d3)

1294

if (!disable_idle_d3)

1291

pci_set_power_state(pdev, PCI_D0);

1295

pci_set_power_state(pdev, PCI_D0);

1292

}

1296

}

1293

1297

1294

static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,

1298

static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,

1295

pci_channel_state_t state)

1299

pci_channel_state_t state)

1296

{

1300

{

1297

struct vfio_pci_device *vdev;

1301

struct vfio_pci_device *vdev;

1298

struct vfio_device *device;

1302

struct vfio_device *device;

1299

1303

1300

device = vfio_device_get_from_dev(&pdev->dev);

1304

device = vfio_device_get_from_dev(&pdev->dev);

1301

if (device == NULL)

1305

if (device == NULL)

1302

return PCI_ERS_RESULT_DISCONNECT;

1306

return PCI_ERS_RESULT_DISCONNECT;

1303

1307

1304

vdev = vfio_device_data(device);

1308

vdev = vfio_device_data(device);

1305

if (vdev == NULL) {

1309

if (vdev == NULL) {

1306

vfio_device_put(device);

1310

vfio_device_put(device);

1307

return PCI_ERS_RESULT_DISCONNECT;

1311

return PCI_ERS_RESULT_DISCONNECT;

1308

}

1312

}

1309

1313

1310

mutex_lock(&vdev->igate);

1314

mutex_lock(&vdev->igate);

1311

1315

1312

if (vdev->err_trigger)

1316

if (vdev->err_trigger)

1313

eventfd_signal(vdev->err_trigger, 1);

1317

eventfd_signal(vdev->err_trigger, 1);

1314

1318

1315

mutex_unlock(&vdev->igate);

1319

mutex_unlock(&vdev->igate);

1316

1320

1317

vfio_device_put(device);

1321

vfio_device_put(device);

1318

1322

1319

return PCI_ERS_RESULT_CAN_RECOVER;

1323

return PCI_ERS_RESULT_CAN_RECOVER;

1320

}

1324

}

1321

1325

1322

static const struct pci_error_handlers vfio_err_handlers = {

1326

static const struct pci_error_handlers vfio_err_handlers = {

1323

.error_detected = vfio_pci_aer_err_detected,

1327

.error_detected = vfio_pci_aer_err_detected,

1324

};

1328

};

1325

1329

1326

static struct pci_driver vfio_pci_driver = {

1330

static struct pci_driver vfio_pci_driver = {

1327

.name = "vfio-pci",

1331

.name = "vfio-pci",

1328

.id_table = NULL, /* only dynamic ids */

1332

.id_table = NULL, /* only dynamic ids */

1329

.probe = vfio_pci_probe,

1333

.probe = vfio_pci_probe,

1330

.remove = vfio_pci_remove,

1334

.remove = vfio_pci_remove,

1331

.err_handler = &vfio_err_handlers,

1335

.err_handler = &vfio_err_handlers,

1332

};

1336

};

1333

1337

1334

struct vfio_devices {

1338

struct vfio_devices {

1335

struct vfio_device **devices;

1339

struct vfio_device **devices;

1336

int cur_index;

1340

int cur_index;

1337

int max_index;

1341

int max_index;

1338

};

1342

};

1339

1343

1340

static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)

1344

static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)

1341

{

1345

{

1342

struct vfio_devices *devs = data;

1346

struct vfio_devices *devs = data;

1343

struct vfio_device *device;

1347

struct vfio_device *device;

1344

1348

1345

if (devs->cur_index == devs->max_index)

1349

if (devs->cur_index == devs->max_index)

1346

return -ENOSPC;

1350

return -ENOSPC;

1347

1351

1348

device = vfio_device_get_from_dev(&pdev->dev);

1352

device = vfio_device_get_from_dev(&pdev->dev);

1349

if (!device)

1353

if (!device)

1350

return -EINVAL;

1354

return -EINVAL;

1351

1355

1352

if (pci_dev_driver(pdev) != &vfio_pci_driver) {

1356

if (pci_dev_driver(pdev) != &vfio_pci_driver) {

1353

vfio_device_put(device);

1357

vfio_device_put(device);

1354

return -EBUSY;

1358

return -EBUSY;

1355

}

1359

}

1356

1360

1357

devs->devices[devs->cur_index++] = device;

1361

devs->devices[devs->cur_index++] = device;

1358

return 0;

1362

return 0;

1359

}

1363

}

1360

1364

1361

/*

1365

/*

1362

* Attempt to do a bus/slot reset if there are devices affected by a reset for

1366

* Attempt to do a bus/slot reset if there are devices affected by a reset for

1363

* this device that are needs_reset and all of the affected devices are unused

1367

* this device that are needs_reset and all of the affected devices are unused

1364

* (!refcnt). Callers are required to hold driver_lock when calling this to

1368

* (!refcnt). Callers are required to hold driver_lock when calling this to

1365

* prevent device opens and concurrent bus reset attempts. We prevent device

1369

* prevent device opens and concurrent bus reset attempts. We prevent device

1366

* unbinds by acquiring and holding a reference to the vfio_device.

1370

* unbinds by acquiring and holding a reference to the vfio_device.

1367

*

1371

*

1368

* NB: vfio-core considers a group to be viable even if some devices are

1372

* NB: vfio-core considers a group to be viable even if some devices are

1369

* bound to drivers like pci-stub or pcieport. Here we require all devices

1373

* bound to drivers like pci-stub or pcieport. Here we require all devices

1370

* to be bound to vfio_pci since that's the only way we can be sure they

1374

* to be bound to vfio_pci since that's the only way we can be sure they

1371

* stay put.

1375

* stay put.

1372

*/

1376

*/

1373

static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)

1377

static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)

1374

{

1378

{

1375

struct vfio_devices devs = { .cur_index = 0 };

1379

struct vfio_devices devs = { .cur_index = 0 };

1376

int i = 0, ret = -EINVAL;

1380

int i = 0, ret = -EINVAL;

1377

bool needs_reset = false, slot = false;

1381

bool needs_reset = false, slot = false;

1378

struct vfio_pci_device *tmp;

1382

struct vfio_pci_device *tmp;

1379

1383

1380

if (!pci_probe_reset_slot(vdev->pdev->slot))

1384

if (!pci_probe_reset_slot(vdev->pdev->slot))

1381

slot = true;

1385

slot = true;

1382

else if (pci_probe_reset_bus(vdev->pdev->bus))

1386

else if (pci_probe_reset_bus(vdev->pdev->bus))

1383

return;

1387

return;

1384

1388

1385

if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,

1389

if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,

1386

&i, slot) || !i)

1390

&i, slot) || !i)

1387

return;

1391

return;

1388

1392

1389

devs.max_index = i;

1393

devs.max_index = i;

1390

devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);

1394

devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);

1391

if (!devs.devices)

1395

if (!devs.devices)

1392

return;

1396

return;

1393

1397

1394

if (vfio_pci_for_each_slot_or_bus(vdev->pdev,

1398

if (vfio_pci_for_each_slot_or_bus(vdev->pdev,

1395

vfio_pci_get_devs, &devs, slot))

1399

vfio_pci_get_devs, &devs, slot))

1396

goto put_devs;

1400

goto put_devs;

1397

1401

1398

for (i = 0; i < devs.cur_index; i++) {

1402

for (i = 0; i < devs.cur_index; i++) {

1399

tmp = vfio_device_data(devs.devices[i]);

1403

tmp = vfio_device_data(devs.devices[i]);

1400

if (tmp->needs_reset)

1404

if (tmp->needs_reset)

1401

needs_reset = true;

1405

needs_reset = true;

1402

if (tmp->refcnt)

1406

if (tmp->refcnt)

1403

goto put_devs;

1407

goto put_devs;

1404

}

1408

}

1405

1409

1406

if (needs_reset)

1410

if (needs_reset)

1407

ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :

1411

ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :

1408

pci_try_reset_bus(vdev->pdev->bus);

1412

pci_try_reset_bus(vdev->pdev->bus);

1409

1413

1410

put_devs:

1414

put_devs:

1411

for (i = 0; i < devs.cur_index; i++) {

1415

for (i = 0; i < devs.cur_index; i++) {

1412

tmp = vfio_device_data(devs.devices[i]);

1416

tmp = vfio_device_data(devs.devices[i]);

1413

if (!ret)

1417

if (!ret)

1414

tmp->needs_reset = false;

1418

tmp->needs_reset = false;

1415

1419

1416

if (!tmp->refcnt && !disable_idle_d3)

1420

if (!tmp->refcnt && !disable_idle_d3)

1417

pci_set_power_state(tmp->pdev, PCI_D3hot);

1421

pci_set_power_state(tmp->pdev, PCI_D3hot);

1418

1422

1419

vfio_device_put(devs.devices[i]);

1423

vfio_device_put(devs.devices[i]);

1420

}

1424

}

1421

1425

1422

kfree(devs.devices);

1426

kfree(devs.devices);

1423

}

1427

}

1424

1428

1425

static void __exit vfio_pci_cleanup(void)

1429

static void __exit vfio_pci_cleanup(void)

1426

{

1430

{

1427

pci_unregister_driver(&vfio_pci_driver);

1431

pci_unregister_driver(&vfio_pci_driver);

1428

vfio_pci_uninit_perm_bits();

1432

vfio_pci_uninit_perm_bits();

1429

}

1433

}

1430

1434

1431

static void __init vfio_pci_fill_ids(void)

1435

static void __init vfio_pci_fill_ids(void)

1432

{

1436

{

1433

char *p, *id;

1437

char *p, *id;

1434

int rc;

1438

int rc;

1435

1439

1436

/* no ids passed actually */

1440

/* no ids passed actually */

1437

if (ids[0] == '\0')

1441

if (ids[0] == '\0')

1438

return;

1442

return;

1439

1443

1440

/* add ids specified in the module parameter */

1444

/* add ids specified in the module parameter */

1441

p = ids;

1445

p = ids;

1442

while ((id = strsep(&p, ","))) {

1446

while ((id = strsep(&p, ","))) {

1443

unsigned int vendor, device, subvendor = PCI_ANY_ID,

1447

unsigned int vendor, device, subvendor = PCI_ANY_ID,

1444

subdevice = PCI_ANY_ID, class = 0, class_mask = 0;

1448

subdevice = PCI_ANY_ID, class = 0, class_mask = 0;

1445

int fields;

1449

int fields;

1446

1450

1447

if (!strlen(id))

1451

if (!strlen(id))

1448

continue;

1452

continue;

1449

1453

1450

fields = sscanf(id, "%x:%x:%x:%x:%x:%x",

1454

fields = sscanf(id, "%x:%x:%x:%x:%x:%x",

1451

&vendor, &device, &subvendor, &subdevice,

1455

&vendor, &device, &subvendor, &subdevice,

1452

&class, &class_mask);

1456

&class, &class_mask);

1453

1457

1454

if (fields < 2) {

1458

if (fields < 2) {

1455

pr_warn("invalid id string \"%s\"\n", id);

1459

pr_warn("invalid id string \"%s\"\n", id);

1456

continue;

1460

continue;

1457

}

1461

}

1458

1462

1459

rc = pci_add_dynid(&vfio_pci_driver, vendor, device,

1463

rc = pci_add_dynid(&vfio_pci_driver, vendor, device,

1460

subvendor, subdevice, class, class_mask, 0);

1464

subvendor, subdevice, class, class_mask, 0);

1461

if (rc)

1465

if (rc)

1462

pr_warn("failed to add dynamic id [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x (%d)\n",

1466

pr_warn("failed to add dynamic id [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x (%d)\n",

1463

vendor, device, subvendor, subdevice,

1467

vendor, device, subvendor, subdevice,

1464

class, class_mask, rc);

1468

class, class_mask, rc);

1465

else

1469

else

1466

pr_info("add [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x\n",

1470

pr_info("add [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x\n",

1467

vendor, device, subvendor, subdevice,

1471

vendor, device, subvendor, subdevice,

1468

class, class_mask);

1472

class, class_mask);

1469

}

1473

}

1470

}

1474

}

1471

1475

1472

static int __init vfio_pci_init(void)

1476

static int __init vfio_pci_init(void)

1473

{

1477

{

1474

int ret;

1478

int ret;

1475

1479

1476

/* Allocate shared config space permision data used by all devices */

1480

/* Allocate shared config space permision data used by all devices */

1477

ret = vfio_pci_init_perm_bits();

1481

ret = vfio_pci_init_perm_bits();

1478

if (ret)

1482

if (ret)

1479

return ret;

1483

return ret;

1480

1484

1481

/* Register and scan for devices */

1485

/* Register and scan for devices */

1482

ret = pci_register_driver(&vfio_pci_driver);

1486

ret = pci_register_driver(&vfio_pci_driver);

1483

if (ret)

1487

if (ret)

1484

goto out_driver;

1488

goto out_driver;

1485

1489

1486

vfio_pci_fill_ids();

1490

vfio_pci_fill_ids();

1487

1491

1488

return 0;

1492

return 0;

1489

1493

1490

out_driver:

1494

out_driver:

1491

vfio_pci_uninit_perm_bits();

1495

vfio_pci_uninit_perm_bits();

1492

return ret;

1496

return ret;

1493

}

1497

}

1494

1498

1495

module_init(vfio_pci_init);

1499

module_init(vfio_pci_init);

1496

module_exit(vfio_pci_cleanup);

1500

module_exit(vfio_pci_cleanup);

1497

1501

1498

MODULE_VERSION(DRIVER_VERSION);

1502

MODULE_VERSION(DRIVER_VERSION);

1499

MODULE_LICENSE("GPL v2");

1503

MODULE_LICENSE("GPL v2");

1500

MODULE_AUTHOR(DRIVER_AUTHOR);

1504

MODULE_AUTHOR(DRIVER_AUTHOR);

1501

MODULE_DESCRIPTION(DRIVER_DESC);

1505

MODULE_DESCRIPTION(DRIVER_DESC);

1502

1506

GITLAB

vfio-pci: Handle error from pci_iomap

 /*
  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  *     Author: Alex Williamson <alex.williamson@redhat.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * Derived from original vfio:
  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  * Author: Tom Lyon, pugs@cisco.com
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/device.h>
 #include <linux/eventfd.h>
 #include <linux/file.h>
 #include <linux/interrupt.h>
 #include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/notifier.h>
 #include <linux/pci.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
 #include <linux/vgaarb.h>
 #include "vfio_pci_private.h"
 #define DRIVER_VERSION  "0.2"
 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
 #define DRIVER_DESC     "VFIO PCI - User Level meta-driver"
 static char ids[1024] __initdata;
 module_param_string(ids, ids, sizeof(ids), 0);
 MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");
 static bool nointxmask;
 module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(nointxmask,
 		  "Disable support for PCI 2.3 style INTx masking.  If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
 #ifdef CONFIG_VFIO_PCI_VGA
 static bool disable_vga;
 module_param(disable_vga, bool, S_IRUGO);
 MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
 #endif
 static bool disable_idle_d3;
 module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(disable_idle_d3,
 		 "Disable using the PCI D3 low power state for idle, unused devices");
 static DEFINE_MUTEX(driver_lock);
 static inline bool vfio_vga_disabled(void)
 {
 #ifdef CONFIG_VFIO_PCI_VGA
 	return disable_vga;
 #else
 	return true;
 #endif
 }
 /*
  * Our VGA arbiter participation is limited since we don't know anything
  * about the device itself.  However, if the device is the only VGA device
  * downstream of a bridge and VFIO VGA support is disabled, then we can
  * safely return legacy VGA IO and memory as not decoded since the user
  * has no way to get to it and routing can be disabled externally at the
  * bridge.
  */
 static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
 {
 	struct vfio_pci_device *vdev = opaque;
 	struct pci_dev *tmp = NULL, *pdev = vdev->pdev;
 	unsigned char max_busnr;
 	unsigned int decodes;
 	if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
 		       VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
 	max_busnr = pci_bus_max_busnr(pdev->bus);
 	decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 	while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
 		if (tmp == pdev ||
 		    pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
 		    pci_is_root_bus(tmp->bus))
 			continue;
 		if (tmp->bus->number >= pdev->bus->number &&
 		    tmp->bus->number <= max_busnr) {
 			pci_dev_put(tmp);
 			decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
 			break;
 		}
 	}
 	return decodes;
 }
 static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
 {
 	return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
 }
 static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
 {
 	struct resource *res;
 	int bar;
 	struct vfio_pci_dummy_resource *dummy_res;
 	INIT_LIST_HEAD(&vdev->dummy_resources_list);
 	for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
 		res = vdev->pdev->resource + bar;
 		if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
 			goto no_mmap;
 		if (!(res->flags & IORESOURCE_MEM))
 			goto no_mmap;
 		/*
 		 * The PCI core shouldn't set up a resource with a
 		 * type but zero size. But there may be bugs that
 		 * cause us to do that.
 		 */
 		if (!resource_size(res))
 			goto no_mmap;
 		if (resource_size(res) >= PAGE_SIZE) {
 			vdev->bar_mmap_supported[bar] = true;
 			continue;
 		}
 		if (!(res->start & ~PAGE_MASK)) {
 			/*
 			 * Add a dummy resource to reserve the remainder
 			 * of the exclusive page in case that hot-add
 			 * device's bar is assigned into it.
 			 */
 			dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
 			if (dummy_res == NULL)
 				goto no_mmap;
 			dummy_res->resource.name = "vfio sub-page reserved";
 			dummy_res->resource.start = res->end + 1;
 			dummy_res->resource.end = res->start + PAGE_SIZE - 1;
 			dummy_res->resource.flags = res->flags;
 			if (request_resource(res->parent,
 						&dummy_res->resource)) {
 				kfree(dummy_res);
 				goto no_mmap;
 			}
 			dummy_res->index = bar;
 			list_add(&dummy_res->res_next,
 					&vdev->dummy_resources_list);
 			vdev->bar_mmap_supported[bar] = true;
 			continue;
 		}
 		/*
 		 * Here we don't handle the case when the BAR is not page
 		 * aligned because we can't expect the BAR will be
 		 * assigned into the same location in a page in guest
 		 * when we passthrough the BAR. And it's hard to access
 		 * this BAR in userspace because we have no way to get
 		 * the BAR's location in a page.
 		 */
 no_mmap:
 		vdev->bar_mmap_supported[bar] = false;
 	}
 }
 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
 static void vfio_pci_disable(struct vfio_pci_device *vdev);
 /*
  * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
  * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
  * If a device implements the former but not the latter we would typically
  * expect broken_intx_masking be set and require an exclusive interrupt.
  * However since we do have control of the device's ability to assert INTx,
  * we can instead pretend that the device does not implement INTx, virtualizing
  * the pin register to report zero and maintaining DisINTx set on the host.
  */
 static bool vfio_pci_nointx(struct pci_dev *pdev)
 {
 	switch (pdev->vendor) {
 	case PCI_VENDOR_ID_INTEL:
 		switch (pdev->device) {
 		/* All i40e (XL710/X710) 10/20/40GbE NICs */
 		case 0x1572:
 		case 0x1574:
 		case 0x1580 ... 0x1581:
 		case 0x1583 ... 0x1589:
 		case 0x37d0 ... 0x37d2:
 			return true;
 		default:
 			return false;
 		}
 	}
 	return false;
 }
 static int vfio_pci_enable(struct vfio_pci_device *vdev)
 {
 	struct pci_dev *pdev = vdev->pdev;
 	int ret;
 	u16 cmd;
 	u8 msix_pos;
 	pci_set_power_state(pdev, PCI_D0);
 	/* Don't allow our initial saved state to include busmaster */
 	pci_clear_master(pdev);
 	ret = pci_enable_device(pdev);
 	if (ret)
 		return ret;
 	vdev->reset_works = (pci_reset_function(pdev) == 0);
 	pci_save_state(pdev);
 	vdev->pci_saved_state = pci_store_saved_state(pdev);
 	if (!vdev->pci_saved_state)
 		pr_debug("%s: Couldn't store %s saved state\n",
 			 __func__, dev_name(&pdev->dev));
 	if (likely(!nointxmask)) {
 		if (vfio_pci_nointx(pdev)) {
 			dev_info(&pdev->dev, "Masking broken INTx support\n");
 			vdev->nointx = true;
 			pci_intx(pdev, 0);
 		} else
 			vdev->pci_2_3 = pci_intx_mask_supported(pdev);
 	}
 	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
 	if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
 		cmd &= ~PCI_COMMAND_INTX_DISABLE;
 		pci_write_config_word(pdev, PCI_COMMAND, cmd);
 	}
 	ret = vfio_config_init(vdev);
 	if (ret) {
 		kfree(vdev->pci_saved_state);
 		vdev->pci_saved_state = NULL;
 		pci_disable_device(pdev);
 		return ret;
 	}
 	msix_pos = pdev->msix_cap;
 	if (msix_pos) {
 		u16 flags;
 		u32 table;
 		pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
 		pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
 		vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
 		vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
 		vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
 	} else
 		vdev->msix_bar = 0xFF;
 	if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
 		vdev->has_vga = true;
 	if (vfio_pci_is_vga(pdev) &&
 	    pdev->vendor == PCI_VENDOR_ID_INTEL &&
 	    IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
 		ret = vfio_pci_igd_init(vdev);
 		if (ret) {
 			dev_warn(&vdev->pdev->dev,
 				 "Failed to setup Intel IGD regions\n");
 			vfio_pci_disable(vdev);
 			return ret;
 		}
 	}
 	vfio_pci_probe_mmaps(vdev);
 	return 0;
 }
 static void vfio_pci_disable(struct vfio_pci_device *vdev)
 {
 	struct pci_dev *pdev = vdev->pdev;
 	struct vfio_pci_dummy_resource *dummy_res, *tmp;
 	int i, bar;
 	/* Stop the device from further DMA */
 	pci_clear_master(pdev);
 	vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
 				VFIO_IRQ_SET_ACTION_TRIGGER,
 				vdev->irq_type, 0, 0, NULL);
 	vdev->virq_disabled = false;
 	for (i = 0; i < vdev->num_regions; i++)
 		vdev->region[i].ops->release(vdev, &vdev->region[i]);
 	vdev->num_regions = 0;
 	kfree(vdev->region);
 	vdev->region = NULL; /* don't krealloc a freed pointer */
 	vfio_config_free(vdev);
 	for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
 		if (!vdev->barmap[bar])
 			continue;
 		pci_iounmap(pdev, vdev->barmap[bar]);
 		pci_release_selected_regions(pdev, 1 << bar);
 		vdev->barmap[bar] = NULL;
 	}
 	list_for_each_entry_safe(dummy_res, tmp,
 				 &vdev->dummy_resources_list, res_next) {
 		list_del(&dummy_res->res_next);
 		release_resource(&dummy_res->resource);
 		kfree(dummy_res);
 	}
 	vdev->needs_reset = true;
 	/*
 	 * If we have saved state, restore it.  If we can reset the device,
 	 * even better.  Resetting with current state seems better than
 	 * nothing, but saving and restoring current state without reset
 	 * is just busy work.
 	 */
 	if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
 		pr_info("%s: Couldn't reload %s saved state\n",
 			__func__, dev_name(&pdev->dev));
 		if (!vdev->reset_works)
 			goto out;
 		pci_save_state(pdev);
 	}
 	/*
 	 * Disable INTx and MSI, presumably to avoid spurious interrupts
 	 * during reset.  Stolen from pci_reset_function()
 	 */
 	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
 	/*
 	 * Try to reset the device.  The success of this is dependent on
 	 * being able to lock the device, which is not always possible.
 	 */
 	if (vdev->reset_works && !pci_try_reset_function(pdev))
 		vdev->needs_reset = false;
 	pci_restore_state(pdev);
 out:
 	pci_disable_device(pdev);
 	vfio_pci_try_bus_reset(vdev);
 	if (!disable_idle_d3)
 		pci_set_power_state(pdev, PCI_D3hot);
 }
 static void vfio_pci_release(void *device_data)
 {
 	struct vfio_pci_device *vdev = device_data;
 	mutex_lock(&driver_lock);
 	if (!(--vdev->refcnt)) {
 		vfio_spapr_pci_eeh_release(vdev->pdev);
 		vfio_pci_disable(vdev);
 	}
 	mutex_unlock(&driver_lock);
 	module_put(THIS_MODULE);
 }
 static int vfio_pci_open(void *device_data)
 {
 	struct vfio_pci_device *vdev = device_data;
 	int ret = 0;
 	if (!try_module_get(THIS_MODULE))
 		return -ENODEV;
 	mutex_lock(&driver_lock);
 	if (!vdev->refcnt) {
 		ret = vfio_pci_enable(vdev);
 		if (ret)
 			goto error;
 		vfio_spapr_pci_eeh_open(vdev->pdev);
 	}
 	vdev->refcnt++;
 error:
 	mutex_unlock(&driver_lock);
 	if (ret)
 		module_put(THIS_MODULE);
 	return ret;
 }
 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
 {
 	if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
 		u8 pin;
 		pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
 		if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin)
 			return 1;
 	} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
 		u8 pos;
 		u16 flags;
 		pos = vdev->pdev->msi_cap;
 		if (pos) {
 			pci_read_config_word(vdev->pdev,
 					     pos + PCI_MSI_FLAGS, &flags);
 			return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
 		}
 	} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
 		u8 pos;
 		u16 flags;
 		pos = vdev->pdev->msix_cap;
 		if (pos) {
 			pci_read_config_word(vdev->pdev,
 					     pos + PCI_MSIX_FLAGS, &flags);
 			return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
 		}
 	} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
 		if (pci_is_pcie(vdev->pdev))
 			return 1;
 	} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
 		return 1;
 	}
 	return 0;
 }
 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
 {
 	(*(int *)data)++;
 	return 0;
 }
 struct vfio_pci_fill_info {
 	int max;
 	int cur;
 	struct vfio_pci_dependent_device *devices;
 };
 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
 {
 	struct vfio_pci_fill_info *fill = data;
 	struct iommu_group *iommu_group;
 	if (fill->cur == fill->max)
 		return -EAGAIN; /* Something changed, try again */
 	iommu_group = iommu_group_get(&pdev->dev);
 	if (!iommu_group)
 		return -EPERM; /* Cannot reset non-isolated devices */
 	fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
 	fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
 	fill->devices[fill->cur].bus = pdev->bus->number;
 	fill->devices[fill->cur].devfn = pdev->devfn;
 	fill->cur++;
 	iommu_group_put(iommu_group);
 	return 0;
 }
 struct vfio_pci_group_entry {
 	struct vfio_group *group;
 	int id;
 };
 struct vfio_pci_group_info {
 	int count;
 	struct vfio_pci_group_entry *groups;
 };
 static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)
 {
 	struct vfio_pci_group_info *info = data;
 	struct iommu_group *group;
 	int id, i;
 	group = iommu_group_get(&pdev->dev);
 	if (!group)
 		return -EPERM;
 	id = iommu_group_id(group);
 	for (i = 0; i < info->count; i++)
 		if (info->groups[i].id == id)
 			break;
 	iommu_group_put(group);
 	return (i == info->count) ? -EINVAL : 0;
 }
 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
 {
 	for (; pdev; pdev = pdev->bus->self)
 		if (pdev->bus == slot->bus)
 			return (pdev->slot == slot);
 	return false;
 }
 struct vfio_pci_walk_info {
 	int (*fn)(struct pci_dev *, void *data);
 	void *data;
 	struct pci_dev *pdev;
 	bool slot;
 	int ret;
 };
 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
 {
 	struct vfio_pci_walk_info *walk = data;
 	if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
 		walk->ret = walk->fn(pdev, walk->data);
 	return walk->ret;
 }
 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
 					 int (*fn)(struct pci_dev *,
 						   void *data), void *data,
 					 bool slot)
 {
 	struct vfio_pci_walk_info walk = {
 		.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
 	};
 	pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
 	return walk.ret;
 }
 static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
 				struct vfio_info_cap *caps)
 {
 	struct vfio_info_cap_header *header;
 	struct vfio_region_info_cap_sparse_mmap *sparse;
 	size_t end, size;
 	int nr_areas = 2, i = 0;
 	end = pci_resource_len(vdev->pdev, vdev->msix_bar);
 	/* If MSI-X table is aligned to the start or end, only one area */
 	if (((vdev->msix_offset & PAGE_MASK) == 0) ||
 	    (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) >= end))
 		nr_areas = 1;
 	size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));
 	header = vfio_info_cap_add(caps, size,
 				   VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
 	if (IS_ERR(header))
 		return PTR_ERR(header);
 	sparse = container_of(header,
 			      struct vfio_region_info_cap_sparse_mmap, header);
 	sparse->nr_areas = nr_areas;
 	if (vdev->msix_offset & PAGE_MASK) {
 		sparse->areas[i].offset = 0;
 		sparse->areas[i].size = vdev->msix_offset & PAGE_MASK;
 		i++;
 	}
 	if (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) < end) {
 		sparse->areas[i].offset = PAGE_ALIGN(vdev->msix_offset +
 						     vdev->msix_size);
 		sparse->areas[i].size = end - sparse->areas[i].offset;
 		i++;
 	}
 	return 0;
 }
 static int region_type_cap(struct vfio_pci_device *vdev,
 			   struct vfio_info_cap *caps,
 			   unsigned int type, unsigned int subtype)
 {
 	struct vfio_info_cap_header *header;
 	struct vfio_region_info_cap_type *cap;
 	header = vfio_info_cap_add(caps, sizeof(*cap),
 				   VFIO_REGION_INFO_CAP_TYPE, 1);
 	if (IS_ERR(header))
 		return PTR_ERR(header);
 	cap = container_of(header, struct vfio_region_info_cap_type, header);
 	cap->type = type;
 	cap->subtype = subtype;
 	return 0;
 }
 int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
 				 unsigned int type, unsigned int subtype,
 				 const struct vfio_pci_regops *ops,
 				 size_t size, u32 flags, void *data)
 {
 	struct vfio_pci_region *region;
 	region = krealloc(vdev->region,
 			  (vdev->num_regions + 1) * sizeof(*region),
 			  GFP_KERNEL);
 	if (!region)
 		return -ENOMEM;
 	vdev->region = region;
 	vdev->region[vdev->num_regions].type = type;
 	vdev->region[vdev->num_regions].subtype = subtype;
 	vdev->region[vdev->num_regions].ops = ops;
 	vdev->region[vdev->num_regions].size = size;
 	vdev->region[vdev->num_regions].flags = flags;
 	vdev->region[vdev->num_regions].data = data;
 	vdev->num_regions++;
 	return 0;
 }
 static long vfio_pci_ioctl(void *device_data,
 			   unsigned int cmd, unsigned long arg)
 {
 	struct vfio_pci_device *vdev = device_data;
 	unsigned long minsz;
 	if (cmd == VFIO_DEVICE_GET_INFO) {
 		struct vfio_device_info info;
 		minsz = offsetofend(struct vfio_device_info, num_irqs);
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (info.argsz < minsz)
 			return -EINVAL;
 		info.flags = VFIO_DEVICE_FLAGS_PCI;
 		if (vdev->reset_works)
 			info.flags |= VFIO_DEVICE_FLAGS_RESET;
 		info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
 		info.num_irqs = VFIO_PCI_NUM_IRQS;
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
 		struct pci_dev *pdev = vdev->pdev;
 		struct vfio_region_info info;
 		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
 		int i, ret;
 		minsz = offsetofend(struct vfio_region_info, offset);
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (info.argsz < minsz)
 			return -EINVAL;
 		switch (info.index) {
 		case VFIO_PCI_CONFIG_REGION_INDEX:
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.size = pdev->cfg_size;
 			info.flags = VFIO_REGION_INFO_FLAG_READ |
 				     VFIO_REGION_INFO_FLAG_WRITE;
 			break;
 		case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.size = pci_resource_len(pdev, info.index);
 			if (!info.size) {
 				info.flags = 0;
 				break;
 			}
 			info.flags = VFIO_REGION_INFO_FLAG_READ |
 				     VFIO_REGION_INFO_FLAG_WRITE;
 			if (vdev->bar_mmap_supported[info.index]) {
 				info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
 				if (info.index == vdev->msix_bar) {
 					ret = msix_sparse_mmap_cap(vdev, &caps);
 					if (ret)
 						return ret;
 				}
 			}
 			break;
 		case VFIO_PCI_ROM_REGION_INDEX:
 		{
 			void __iomem *io;
 			size_t size;
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.flags = 0;
 			/* Report the BAR size, not the ROM size */
 			info.size = pci_resource_len(pdev, info.index);
 			if (!info.size) {
 				/* Shadow ROMs appear as PCI option ROMs */
 				if (pdev->resource[PCI_ROM_RESOURCE].flags &
 							IORESOURCE_ROM_SHADOW)
 					info.size = 0x20000;
 				else
 					break;
 			}
 			/* Is it really there? */
 			io = pci_map_rom(pdev, &size);
 			if (!io || !size) {
 				info.size = 0;
 				break;
 			}
 			pci_unmap_rom(pdev, io);
 			info.flags = VFIO_REGION_INFO_FLAG_READ;
 			break;
 		}
 		case VFIO_PCI_VGA_REGION_INDEX:
 			if (!vdev->has_vga)
 				return -EINVAL;
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.size = 0xc0000;
 			info.flags = VFIO_REGION_INFO_FLAG_READ |
 				     VFIO_REGION_INFO_FLAG_WRITE;
 			break;
 		default:
 			if (info.index >=
 			    VFIO_PCI_NUM_REGIONS + vdev->num_regions)
 				return -EINVAL;
 			i = info.index - VFIO_PCI_NUM_REGIONS;
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.size = vdev->region[i].size;
 			info.flags = vdev->region[i].flags;
 			ret = region_type_cap(vdev, &caps,
 					      vdev->region[i].type,
 					      vdev->region[i].subtype);
 			if (ret)
 				return ret;
 		}
 		if (caps.size) {
 			info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
 			if (info.argsz < sizeof(info) + caps.size) {
 				info.argsz = sizeof(info) + caps.size;
 				info.cap_offset = 0;
 			} else {
 				vfio_info_cap_shift(&caps, sizeof(info));
 				if (copy_to_user((void __user *)arg +
 						  sizeof(info), caps.buf,
 						  caps.size)) {
 					kfree(caps.buf);
 					return -EFAULT;
 				}
 				info.cap_offset = sizeof(info);
 			}
 			kfree(caps.buf);
 		}
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
 		minsz = offsetofend(struct vfio_irq_info, count);
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
 			return -EINVAL;
 		switch (info.index) {
 		case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
 		case VFIO_PCI_REQ_IRQ_INDEX:
 			break;
 		case VFIO_PCI_ERR_IRQ_INDEX:
 			if (pci_is_pcie(vdev->pdev))
 				break;
 		/* pass thru to return error */
 		default:
 			return -EINVAL;
 		}
 		info.flags = VFIO_IRQ_INFO_EVENTFD;
 		info.count = vfio_pci_get_irq_count(vdev, info.index);
 		if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
 			info.flags |= (VFIO_IRQ_INFO_MASKABLE |
 				       VFIO_IRQ_INFO_AUTOMASKED);
 		else
 			info.flags |= VFIO_IRQ_INFO_NORESIZE;
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
 		struct vfio_irq_set hdr;
 		size_t size;
 		u8 *data = NULL;
 		int max, ret = 0;
 		minsz = offsetofend(struct vfio_irq_set, count);
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
 		    hdr.count >= (U32_MAX - hdr.start) ||
 		    hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
 				  VFIO_IRQ_SET_ACTION_TYPE_MASK))
 			return -EINVAL;
 		max = vfio_pci_get_irq_count(vdev, hdr.index);
 		if (hdr.start >= max || hdr.start + hdr.count > max)
 			return -EINVAL;
 		switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
 		case VFIO_IRQ_SET_DATA_NONE:
 			size = 0;
 			break;
 		case VFIO_IRQ_SET_DATA_BOOL:
 			size = sizeof(uint8_t);
 			break;
 		case VFIO_IRQ_SET_DATA_EVENTFD:
 			size = sizeof(int32_t);
 			break;
 		default:
 			return -EINVAL;
 		}
 		if (size) {
 			if (hdr.argsz - minsz < hdr.count * size)
 				return -EINVAL;
 			data = memdup_user((void __user *)(arg + minsz),
 					   hdr.count * size);
 			if (IS_ERR(data))
 				return PTR_ERR(data);
 		}
 		mutex_lock(&vdev->igate);
 		ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
 					      hdr.start, hdr.count, data);
 		mutex_unlock(&vdev->igate);
 		kfree(data);
 		return ret;
 	} else if (cmd == VFIO_DEVICE_RESET) {
 		return vdev->reset_works ?
 			pci_try_reset_function(vdev->pdev) : -EINVAL;
 	} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
 		struct vfio_pci_hot_reset_info hdr;
 		struct vfio_pci_fill_info fill = { 0 };
 		struct vfio_pci_dependent_device *devices = NULL;
 		bool slot = false;
 		int ret = 0;
 		minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (hdr.argsz < minsz)
 			return -EINVAL;
 		hdr.flags = 0;
 		/* Can we do a slot or bus reset or neither? */
 		if (!pci_probe_reset_slot(vdev->pdev->slot))
 			slot = true;
 		else if (pci_probe_reset_bus(vdev->pdev->bus))
 			return -ENODEV;
 		/* How many devices are affected? */
 		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
 						    vfio_pci_count_devs,
 						    &fill.max, slot);
 		if (ret)
 			return ret;
 		WARN_ON(!fill.max); /* Should always be at least one */
 		/*
 		 * If there's enough space, fill it now, otherwise return
 		 * -ENOSPC and the number of devices affected.
 		 */
 		if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
 			ret = -ENOSPC;
 			hdr.count = fill.max;
 			goto reset_info_exit;
 		}
 		devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
 		if (!devices)
 			return -ENOMEM;
 		fill.devices = devices;
 		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
 						    vfio_pci_fill_devs,
 						    &fill, slot);
 		/*
 		 * If a device was removed between counting and filling,
 		 * we may come up short of fill.max.  If a device was
 		 * added, we'll have a return of -EAGAIN above.
 		 */
 		if (!ret)
 			hdr.count = fill.cur;
 reset_info_exit:
 		if (copy_to_user((void __user *)arg, &hdr, minsz))
 			ret = -EFAULT;
 		if (!ret) {
 			if (copy_to_user((void __user *)(arg + minsz), devices,
 					 hdr.count * sizeof(*devices)))
 				ret = -EFAULT;
 		}
 		kfree(devices);
 		return ret;
 	} else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
 		struct vfio_pci_hot_reset hdr;
 		int32_t *group_fds;
 		struct vfio_pci_group_entry *groups;
 		struct vfio_pci_group_info info;
 		bool slot = false;
 		int i, count = 0, ret = 0;
 		minsz = offsetofend(struct vfio_pci_hot_reset, count);
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 		if (hdr.argsz < minsz || hdr.flags)
 			return -EINVAL;
 		/* Can we do a slot or bus reset or neither? */
 		if (!pci_probe_reset_slot(vdev->pdev->slot))
 			slot = true;
 		else if (pci_probe_reset_bus(vdev->pdev->bus))
 			return -ENODEV;
 		/*
 		 * We can't let userspace give us an arbitrarily large
 		 * buffer to copy, so verify how many we think there
 		 * could be.  Note groups can have multiple devices so
 		 * one group per device is the max.
 		 */
 		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
 						    vfio_pci_count_devs,
 						    &count, slot);
 		if (ret)
 			return ret;
 		/* Somewhere between 1 and count is OK */
 		if (!hdr.count || hdr.count > count)
 			return -EINVAL;
 		group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
 		groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
 		if (!group_fds || !groups) {
 			kfree(group_fds);
 			kfree(groups);
 			return -ENOMEM;
 		}
 		if (copy_from_user(group_fds, (void __user *)(arg + minsz),
 				   hdr.count * sizeof(*group_fds))) {
 			kfree(group_fds);
 			kfree(groups);
 			return -EFAULT;
 		}
 		/*
 		 * For each group_fd, get the group through the vfio external
 		 * user interface and store the group and iommu ID.  This
 		 * ensures the group is held across the reset.
 		 */
 		for (i = 0; i < hdr.count; i++) {
 			struct vfio_group *group;
 			struct fd f = fdget(group_fds[i]);
 			if (!f.file) {
 				ret = -EBADF;
 				break;
 			}
 			group = vfio_group_get_external_user(f.file);
 			fdput(f);
 			if (IS_ERR(group)) {
 				ret = PTR_ERR(group);
 				break;
 			}
 			groups[i].group = group;
 			groups[i].id = vfio_external_user_iommu_id(group);
 		}
 		kfree(group_fds);
 		/* release reference to groups on error */
 		if (ret)
 			goto hot_reset_release;
 		info.count = hdr.count;
 		info.groups = groups;
 		/*
 		 * Test whether all the affected devices are contained
 		 * by the set of groups provided by the user.
 		 */
 		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
 						    vfio_pci_validate_devs,
 						    &info, slot);
 		if (!ret)
 			/* User has access, do the reset */
 			ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
 				     pci_try_reset_bus(vdev->pdev->bus);
 hot_reset_release:
 		for (i--; i >= 0; i--)
 			vfio_group_put_external_user(groups[i].group);
 		kfree(groups);
 		return ret;
 	}
 	return -ENOTTY;
 }
 static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
 			   size_t count, loff_t *ppos, bool iswrite)
 {
 	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 	struct vfio_pci_device *vdev = device_data;
 	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
 		return -EINVAL;
 	switch (index) {
 	case VFIO_PCI_CONFIG_REGION_INDEX:
 		return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
 	case VFIO_PCI_ROM_REGION_INDEX:
 		if (iswrite)
 			return -EINVAL;
 		return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
 	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 		return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
 	case VFIO_PCI_VGA_REGION_INDEX:
 		return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
 	default:
 		index -= VFIO_PCI_NUM_REGIONS;
 		return vdev->region[index].ops->rw(vdev, buf,
 						   count, ppos, iswrite);
 	}
 	return -EINVAL;
 }
 static ssize_t vfio_pci_read(void *device_data, char __user *buf,
 			     size_t count, loff_t *ppos)
 {
 	if (!count)
 		return 0;
 	return vfio_pci_rw(device_data, buf, count, ppos, false);
 }
 static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
 			      size_t count, loff_t *ppos)
 {
 	if (!count)
 		return 0;
 	return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
 }
 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
 {
 	struct vfio_pci_device *vdev = device_data;
 	struct pci_dev *pdev = vdev->pdev;
 	unsigned int index;
 	u64 phys_len, req_len, pgoff, req_start;
 	int ret;
 	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
 	if (vma->vm_end < vma->vm_start)
 		return -EINVAL;
 	if ((vma->vm_flags & VM_SHARED) == 0)
 		return -EINVAL;
 	if (index >= VFIO_PCI_ROM_REGION_INDEX)
 		return -EINVAL;
 	if (!vdev->bar_mmap_supported[index])
 		return -EINVAL;
 	phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
 	req_len = vma->vm_end - vma->vm_start;
 	pgoff = vma->vm_pgoff &
 		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
 	req_start = pgoff << PAGE_SHIFT;
 	if (req_start + req_len > phys_len)
 		return -EINVAL;
 	if (index == vdev->msix_bar) {
 		/*
 		 * Disallow mmaps overlapping the MSI-X table; users don't
 		 * get to touch this directly.  We could find somewhere
 		 * else to map the overlap, but page granularity is only
 		 * a recommendation, not a requirement, so the user needs
 		 * to know which bits are real.  Requiring them to mmap
 		 * around the table makes that clear.
 		 */
 		/* If neither entirely above nor below, then it overlaps */
 		if (!(req_start >= vdev->msix_offset + vdev->msix_size ||
 		      req_start + req_len <= vdev->msix_offset))
 			return -EINVAL;
 	}
 	/*
 	 * Even though we don't make use of the barmap for the mmap,
 	 * we need to request the region and the barmap tracks that.
 	 */
 	if (!vdev->barmap[index]) {
 		ret = pci_request_selected_regions(pdev,
 						   1 << index, "vfio-pci");
 		if (ret)
 			return ret;
 		vdev->barmap[index] = pci_iomap(pdev, index, 0);
+		if (!vdev->barmap[index]) {
+			pci_release_selected_regions(pdev, 1 << index);
+			return -ENOMEM;
+		}
 	}
 	vma->vm_private_data = vdev;
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
 	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 			       req_len, vma->vm_page_prot);
 }
 static void vfio_pci_request(void *device_data, unsigned int count)
 {
 	struct vfio_pci_device *vdev = device_data;
 	mutex_lock(&vdev->igate);
 	if (vdev->req_trigger) {
 		if (!(count % 10))
 			dev_notice_ratelimited(&vdev->pdev->dev,
 				"Relaying device request to user (#%u)\n",
 				count);
 		eventfd_signal(vdev->req_trigger, 1);
 	} else if (count == 0) {
 		dev_warn(&vdev->pdev->dev,
 			"No device request channel registered, blocked until released by user\n");
 	}
 	mutex_unlock(&vdev->igate);
 }
 static const struct vfio_device_ops vfio_pci_ops = {
 	.name		= "vfio-pci",
 	.open		= vfio_pci_open,
 	.release	= vfio_pci_release,
 	.ioctl		= vfio_pci_ioctl,
 	.read		= vfio_pci_read,
 	.write		= vfio_pci_write,
 	.mmap		= vfio_pci_mmap,
 	.request	= vfio_pci_request,
 };
 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct vfio_pci_device *vdev;
 	struct iommu_group *group;
 	int ret;
 	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 		return -EINVAL;
 	group = vfio_iommu_group_get(&pdev->dev);
 	if (!group)
 		return -EINVAL;
 	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 	if (!vdev) {
 		vfio_iommu_group_put(group, &pdev->dev);
 		return -ENOMEM;
 	}
 	vdev->pdev = pdev;
 	vdev->irq_type = VFIO_PCI_NUM_IRQS;
 	mutex_init(&vdev->igate);
 	spin_lock_init(&vdev->irqlock);
 	ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
 	if (ret) {
 		vfio_iommu_group_put(group, &pdev->dev);
 		kfree(vdev);
 		return ret;
 	}
 	if (vfio_pci_is_vga(pdev)) {
 		vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
 		vga_set_legacy_decoding(pdev,
 					vfio_pci_set_vga_decode(vdev, false));
 	}
 	if (!disable_idle_d3) {
 		/*
 		 * pci-core sets the device power state to an unknown value at
 		 * bootup and after being removed from a driver.  The only
 		 * transition it allows from this unknown state is to D0, which
 		 * typically happens when a driver calls pci_enable_device().
 		 * We're not ready to enable the device yet, but we do want to
 		 * be able to get to D3.  Therefore first do a D0 transition
 		 * before going to D3.
 		 */
 		pci_set_power_state(pdev, PCI_D0);
 		pci_set_power_state(pdev, PCI_D3hot);
 	}
 	return ret;
 }
 static void vfio_pci_remove(struct pci_dev *pdev)
 {
 	struct vfio_pci_device *vdev;
 	vdev = vfio_del_group_dev(&pdev->dev);
 	if (!vdev)
 		return;
 	vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
 	kfree(vdev->region);
 	kfree(vdev);
 	if (vfio_pci_is_vga(pdev)) {
 		vga_client_register(pdev, NULL, NULL, NULL);
 		vga_set_legacy_decoding(pdev,
 				VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
 				VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);
 	}
 	if (!disable_idle_d3)
 		pci_set_power_state(pdev, PCI_D0);
 }
 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
 						  pci_channel_state_t state)
 {
 	struct vfio_pci_device *vdev;
 	struct vfio_device *device;
 	device = vfio_device_get_from_dev(&pdev->dev);
 	if (device == NULL)
 		return PCI_ERS_RESULT_DISCONNECT;
 	vdev = vfio_device_data(device);
 	if (vdev == NULL) {
 		vfio_device_put(device);
 		return PCI_ERS_RESULT_DISCONNECT;
 	}
 	mutex_lock(&vdev->igate);
 	if (vdev->err_trigger)
 		eventfd_signal(vdev->err_trigger, 1);
 	mutex_unlock(&vdev->igate);
 	vfio_device_put(device);
 	return PCI_ERS_RESULT_CAN_RECOVER;
 }
 static const struct pci_error_handlers vfio_err_handlers = {
 	.error_detected = vfio_pci_aer_err_detected,
 };
 static struct pci_driver vfio_pci_driver = {
 	.name		= "vfio-pci",
 	.id_table	= NULL, /* only dynamic ids */
 	.probe		= vfio_pci_probe,
 	.remove		= vfio_pci_remove,
 	.err_handler	= &vfio_err_handlers,
 };
 struct vfio_devices {
 	struct vfio_device **devices;
 	int cur_index;
 	int max_index;
 };
 static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)
 {
 	struct vfio_devices *devs = data;
 	struct vfio_device *device;
 	if (devs->cur_index == devs->max_index)
 		return -ENOSPC;
 	device = vfio_device_get_from_dev(&pdev->dev);
 	if (!device)
 		return -EINVAL;
 	if (pci_dev_driver(pdev) != &vfio_pci_driver) {
 		vfio_device_put(device);
 		return -EBUSY;
 	}
 	devs->devices[devs->cur_index++] = device;
 	return 0;
 }
 /*
  * Attempt to do a bus/slot reset if there are devices affected by a reset for
  * this device that are needs_reset and all of the affected devices are unused
  * (!refcnt).  Callers are required to hold driver_lock when calling this to
  * prevent device opens and concurrent bus reset attempts.  We prevent device
  * unbinds by acquiring and holding a reference to the vfio_device.
  *
  * NB: vfio-core considers a group to be viable even if some devices are
  * bound to drivers like pci-stub or pcieport.  Here we require all devices
  * to be bound to vfio_pci since that's the only way we can be sure they
  * stay put.
  */
 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
 {
 	struct vfio_devices devs = { .cur_index = 0 };
 	int i = 0, ret = -EINVAL;
 	bool needs_reset = false, slot = false;
 	struct vfio_pci_device *tmp;
 	if (!pci_probe_reset_slot(vdev->pdev->slot))
 		slot = true;
 	else if (pci_probe_reset_bus(vdev->pdev->bus))
 		return;
 	if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
 					  &i, slot) || !i)
 		return;
 	devs.max_index = i;
 	devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);
 	if (!devs.devices)
 		return;
 	if (vfio_pci_for_each_slot_or_bus(vdev->pdev,
 					  vfio_pci_get_devs, &devs, slot))
 		goto put_devs;
 	for (i = 0; i < devs.cur_index; i++) {
 		tmp = vfio_device_data(devs.devices[i]);
 		if (tmp->needs_reset)
 			needs_reset = true;
 		if (tmp->refcnt)
 			goto put_devs;
 	}
 	if (needs_reset)
 		ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
 			     pci_try_reset_bus(vdev->pdev->bus);
 put_devs:
 	for (i = 0; i < devs.cur_index; i++) {
 		tmp = vfio_device_data(devs.devices[i]);
 		if (!ret)
 			tmp->needs_reset = false;
 		if (!tmp->refcnt && !disable_idle_d3)
 			pci_set_power_state(tmp->pdev, PCI_D3hot);
 		vfio_device_put(devs.devices[i]);
 	}
 	kfree(devs.devices);
 }
 static void __exit vfio_pci_cleanup(void)
 {
 	pci_unregister_driver(&vfio_pci_driver);
 	vfio_pci_uninit_perm_bits();
 }
 static void __init vfio_pci_fill_ids(void)
 {
 	char *p, *id;
 	int rc;
 	/* no ids passed actually */
 	if (ids[0] == '\0')
 		return;
 	/* add ids specified in the module parameter */
 	p = ids;
 	while ((id = strsep(&p, ","))) {
 		unsigned int vendor, device, subvendor = PCI_ANY_ID,
 			subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
 		int fields;
 		if (!strlen(id))
 			continue;
 		fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
 				&vendor, &device, &subvendor, &subdevice,
 				&class, &class_mask);
 		if (fields < 2) {
 			pr_warn("invalid id string \"%s\"\n", id);
 			continue;
 		}
 		rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
 				   subvendor, subdevice, class, class_mask, 0);
 		if (rc)
 			pr_warn("failed to add dynamic id [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x (%d)\n",
 				vendor, device, subvendor, subdevice,
 				class, class_mask, rc);
 		else
 			pr_info("add [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x\n",
 				vendor, device, subvendor, subdevice,
 				class, class_mask);
 	}
 }
 static int __init vfio_pci_init(void)
 {
 	int ret;
 	/* Allocate shared config space permision data used by all devices */
 	ret = vfio_pci_init_perm_bits();
 	if (ret)
 		return ret;
 	/* Register and scan for devices */
 	ret = pci_register_driver(&vfio_pci_driver);
 	if (ret)
 		goto out_driver;
 	vfio_pci_fill_ids();
 	return 0;
 out_driver:
 	vfio_pci_uninit_perm_bits();
 	return ret;
 }
 module_init(vfio_pci_init);
 module_exit(vfio_pci_cleanup);
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);