Doug / smarc-fsl-linux-kernel

1

/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.

1

/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.

2

*

2

*

3

4

*/

4

*/

5

6

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

6

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

7

8

#include <linux/kernel.h>

8

#include <linux/kernel.h>

9

#include <linux/module.h>

9

#include <linux/module.h>

10

#include <linux/of.h>

10

#include <linux/of.h>

11

#include <linux/of_device.h>

11

#include <linux/of_device.h>

12

#include <linux/cpumask.h>

12

#include <linux/cpumask.h>

13

#include <linux/slab.h>

13

#include <linux/slab.h>

14

#include <linux/interrupt.h>

14

#include <linux/interrupt.h>

15

#include <linux/crypto.h>

15

#include <linux/crypto.h>

16

#include <crypto/md5.h>

16

#include <crypto/md5.h>

17

#include <crypto/sha.h>

17

#include <crypto/sha.h>

18

#include <crypto/aes.h>

18

#include <crypto/aes.h>

19

#include <crypto/des.h>

19

#include <crypto/des.h>

20

#include <linux/mutex.h>

20

#include <linux/mutex.h>

21

#include <linux/delay.h>

21

#include <linux/delay.h>

22

#include <linux/sched.h>

22

#include <linux/sched.h>

23

24

#include <crypto/internal/hash.h>

24

#include <crypto/internal/hash.h>

25

#include <crypto/scatterwalk.h>

25

#include <crypto/scatterwalk.h>

26

#include <crypto/algapi.h>

26

#include <crypto/algapi.h>

27

28

#include <asm/hypervisor.h>

28

#include <asm/hypervisor.h>

29

#include <asm/mdesc.h>

29

#include <asm/mdesc.h>

30

31

#include "n2_core.h"

31

#include "n2_core.h"

32

33

#define DRV_MODULE_NAME "n2_crypto"

33

#define DRV_MODULE_NAME "n2_crypto"

34

#define DRV_MODULE_VERSION "0.2"

34

#define DRV_MODULE_VERSION "0.2"

35

#define DRV_MODULE_RELDATE "July 28, 2011"

35

#define DRV_MODULE_RELDATE "July 28, 2011"

36

37

static char version[] __devinitdata =

37

static char version[] __devinitdata =

38

DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

38

DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

39

40

MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");

40

MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");

41

MODULE_DESCRIPTION("Niagara2 Crypto driver");

41

MODULE_DESCRIPTION("Niagara2 Crypto driver");

42

MODULE_LICENSE("GPL");

42

MODULE_LICENSE("GPL");

43

MODULE_VERSION(DRV_MODULE_VERSION);

43

MODULE_VERSION(DRV_MODULE_VERSION);

44

45

#define N2_CRA_PRIORITY 300

45

#define N2_CRA_PRIORITY 300

46

47

static DEFINE_MUTEX(spu_lock);

47

static DEFINE_MUTEX(spu_lock);

48

49

struct spu_queue {

49

struct spu_queue {

50

cpumask_t sharing;

50

cpumask_t sharing;

51

unsigned long qhandle;

51

unsigned long qhandle;

52

53

spinlock_t lock;

53

spinlock_t lock;

54

u8 q_type;

54

u8 q_type;

55

void *q;

55

void *q;

56

unsigned long head;

56

unsigned long head;

57

unsigned long tail;

57

unsigned long tail;

58

struct list_head jobs;

58

struct list_head jobs;

59

60

unsigned long devino;

60

unsigned long devino;

61

62

char irq_name[32];

62

char irq_name[32];

63

unsigned int irq;

63

unsigned int irq;

64

65

struct list_head list;

65

struct list_head list;

66

};

66

};

67

68

static struct spu_queue **cpu_to_cwq;

68

static struct spu_queue **cpu_to_cwq;

69

static struct spu_queue **cpu_to_mau;

69

static struct spu_queue **cpu_to_mau;

70

71

static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)

71

static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)

72

{

72

{

73

if (q->q_type == HV_NCS_QTYPE_MAU) {

73

if (q->q_type == HV_NCS_QTYPE_MAU) {

74

off += MAU_ENTRY_SIZE;

74

off += MAU_ENTRY_SIZE;

75

if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))

75

if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))

76

off = 0;

76

off = 0;

77

} else {

77

} else {

78

off += CWQ_ENTRY_SIZE;

78

off += CWQ_ENTRY_SIZE;

79

if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))

79

if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))

80

off = 0;

80

off = 0;

81

}

81

}

82

return off;

82

return off;

83

}

83

}

84

85

struct n2_request_common {

85

struct n2_request_common {

86

struct list_head entry;

86

struct list_head entry;

87

unsigned int offset;

87

unsigned int offset;

88

};

88

};

89

#define OFFSET_NOT_RUNNING (~(unsigned int)0)

89

#define OFFSET_NOT_RUNNING (~(unsigned int)0)

90

91

/* An async job request records the final tail value it used in

91

/* An async job request records the final tail value it used in

92

* n2_request_common->offset, test to see if that offset is in

92

* n2_request_common->offset, test to see if that offset is in

93

* the range old_head, new_head, inclusive.

93

* the range old_head, new_head, inclusive.

94

*/

94

*/

95

static inline bool job_finished(struct spu_queue *q, unsigned int offset,

95

static inline bool job_finished(struct spu_queue *q, unsigned int offset,

96

unsigned long old_head, unsigned long new_head)

96

unsigned long old_head, unsigned long new_head)

97

{

97

{

98

if (old_head <= new_head) {

98

if (old_head <= new_head) {

99

if (offset > old_head && offset <= new_head)

99

if (offset > old_head && offset <= new_head)

100

return true;

100

return true;

101

} else {

101

} else {

102

if (offset > old_head || offset <= new_head)

102

if (offset > old_head || offset <= new_head)

103

return true;

103

return true;

104

}

104

}

105

return false;

105

return false;

106

}

106

}

107

108

/* When the HEAD marker is unequal to the actual HEAD, we get

108

/* When the HEAD marker is unequal to the actual HEAD, we get

109

* a virtual device INO interrupt. We should process the

109

* a virtual device INO interrupt. We should process the

110

* completed CWQ entries and adjust the HEAD marker to clear

110

* completed CWQ entries and adjust the HEAD marker to clear

111

* the IRQ.

111

* the IRQ.

112

*/

112

*/

113

static irqreturn_t cwq_intr(int irq, void *dev_id)

113

static irqreturn_t cwq_intr(int irq, void *dev_id)

114

{

114

{

115

unsigned long off, new_head, hv_ret;

115

unsigned long off, new_head, hv_ret;

116

struct spu_queue *q = dev_id;

116

struct spu_queue *q = dev_id;

117

118

pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",

118

pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",

119

smp_processor_id(), q->qhandle);

119

smp_processor_id(), q->qhandle);

120

121

spin_lock(&q->lock);

121

spin_lock(&q->lock);

122

123

hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);

123

hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);

124

125

pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",

125

pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",

126

smp_processor_id(), new_head, hv_ret);

126

smp_processor_id(), new_head, hv_ret);

127

128

for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {

128

for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {

129

/* XXX ... XXX */

129

/* XXX ... XXX */

130

}

130

}

131

132

hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);

132

hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);

133

if (hv_ret == HV_EOK)

133

if (hv_ret == HV_EOK)

134

q->head = new_head;

134

q->head = new_head;

135

136

spin_unlock(&q->lock);

136

spin_unlock(&q->lock);

137

138

return IRQ_HANDLED;

138

return IRQ_HANDLED;

139

}

139

}

140

141

static irqreturn_t mau_intr(int irq, void *dev_id)

141

static irqreturn_t mau_intr(int irq, void *dev_id)

142

{

142

{

143

struct spu_queue *q = dev_id;

143

struct spu_queue *q = dev_id;

144

unsigned long head, hv_ret;

144

unsigned long head, hv_ret;

145

146

spin_lock(&q->lock);

146

spin_lock(&q->lock);

147

148

pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",

148

pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",

149

smp_processor_id(), q->qhandle);

149

smp_processor_id(), q->qhandle);

150

151

hv_ret = sun4v_ncs_gethead(q->qhandle, &head);

151

hv_ret = sun4v_ncs_gethead(q->qhandle, &head);

152

153

pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",

153

pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",

154

smp_processor_id(), head, hv_ret);

154

smp_processor_id(), head, hv_ret);

155

156

sun4v_ncs_sethead_marker(q->qhandle, head);

156

sun4v_ncs_sethead_marker(q->qhandle, head);

157

158

spin_unlock(&q->lock);

158

spin_unlock(&q->lock);

159

160

return IRQ_HANDLED;

160

return IRQ_HANDLED;

161

}

161

}

162

163

static void *spu_queue_next(struct spu_queue *q, void *cur)

163

static void *spu_queue_next(struct spu_queue *q, void *cur)

164

{

164

{

165

return q->q + spu_next_offset(q, cur - q->q);

165

return q->q + spu_next_offset(q, cur - q->q);

166

}

166

}

167

168

static int spu_queue_num_free(struct spu_queue *q)

168

static int spu_queue_num_free(struct spu_queue *q)

169

{

169

{

170

unsigned long head = q->head;

170

unsigned long head = q->head;

171

unsigned long tail = q->tail;

171

unsigned long tail = q->tail;

172

unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);

172

unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);

173

unsigned long diff;

173

unsigned long diff;

174

175

if (head > tail)

175

if (head > tail)

176

diff = head - tail;

176

diff = head - tail;

177

else

177

else

178

diff = (end - tail) + head;

178

diff = (end - tail) + head;

179

180

return (diff / CWQ_ENTRY_SIZE) - 1;

180

return (diff / CWQ_ENTRY_SIZE) - 1;

181

}

181

}

182

183

static void *spu_queue_alloc(struct spu_queue *q, int num_entries)

183

static void *spu_queue_alloc(struct spu_queue *q, int num_entries)

184

{

184

{

185

int avail = spu_queue_num_free(q);

185

int avail = spu_queue_num_free(q);

186

187

if (avail >= num_entries)

187

if (avail >= num_entries)

188

return q->q + q->tail;

188

return q->q + q->tail;

189

190

return NULL;

190

return NULL;

191

}

191

}

192

193

static unsigned long spu_queue_submit(struct spu_queue *q, void *last)

193

static unsigned long spu_queue_submit(struct spu_queue *q, void *last)

194

{

194

{

195

unsigned long hv_ret, new_tail;

195

unsigned long hv_ret, new_tail;

196

197

new_tail = spu_next_offset(q, last - q->q);

197

new_tail = spu_next_offset(q, last - q->q);

198

199

hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);

199

hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);

200

if (hv_ret == HV_EOK)

200

if (hv_ret == HV_EOK)

201

q->tail = new_tail;

201

q->tail = new_tail;

202

return hv_ret;

202

return hv_ret;

203

}

203

}

204

205

static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,

205

static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,

206

int enc_type, int auth_type,

206

int enc_type, int auth_type,

207

unsigned int hash_len,

207

unsigned int hash_len,

208

bool sfas, bool sob, bool eob, bool encrypt,

208

bool sfas, bool sob, bool eob, bool encrypt,

209

int opcode)

209

int opcode)

210

{

210

{

211

u64 word = (len - 1) & CONTROL_LEN;

211

u64 word = (len - 1) & CONTROL_LEN;

212

213

word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);

213

word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);

214

word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);

214

word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);

215

word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);

215

word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);

216

if (sfas)

216

if (sfas)

217

word |= CONTROL_STORE_FINAL_AUTH_STATE;

217

word |= CONTROL_STORE_FINAL_AUTH_STATE;

218

if (sob)

218

if (sob)

219

word |= CONTROL_START_OF_BLOCK;

219

word |= CONTROL_START_OF_BLOCK;

220

if (eob)

220

if (eob)

221

word |= CONTROL_END_OF_BLOCK;

221

word |= CONTROL_END_OF_BLOCK;

222

if (encrypt)

222

if (encrypt)

223

word |= CONTROL_ENCRYPT;

223

word |= CONTROL_ENCRYPT;

224

if (hmac_key_len)

224

if (hmac_key_len)

225

word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;

225

word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;

226

if (hash_len)

226

if (hash_len)

227

word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;

227

word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;

228

229

return word;

229

return word;

230

}

230

}

231

232

#if 0

232

#if 0

233

static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)

233

static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)

234

{

234

{

235

if (this_len >= 64 ||

235

if (this_len >= 64 ||

236

qp->head != qp->tail)

236

qp->head != qp->tail)

237

return true;

237

return true;

238

return false;

238

return false;

239

}

239

}

240

#endif

240

#endif

241

242

struct n2_ahash_alg {

242

struct n2_ahash_alg {

243

struct list_head entry;

243

struct list_head entry;

244

const char *hash_zero;

244

const char *hash_zero;

245

const u32 *hash_init;

245

const u32 *hash_init;

246

u8 hw_op_hashsz;

246

u8 hw_op_hashsz;

247

u8 digest_size;

247

u8 digest_size;

248

u8 auth_type;

248

u8 auth_type;

249

u8 hmac_type;

249

u8 hmac_type;

250

struct ahash_alg alg;

250

struct ahash_alg alg;

251

};

251

};

252

253

static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)

253

static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)

254

{

254

{

255

struct crypto_alg *alg = tfm->__crt_alg;

255

struct crypto_alg *alg = tfm->__crt_alg;

256

struct ahash_alg *ahash_alg;

256

struct ahash_alg *ahash_alg;

257

258

ahash_alg = container_of(alg, struct ahash_alg, halg.base);

258

ahash_alg = container_of(alg, struct ahash_alg, halg.base);

259

260

return container_of(ahash_alg, struct n2_ahash_alg, alg);

260

return container_of(ahash_alg, struct n2_ahash_alg, alg);

261

}

261

}

262

263

struct n2_hmac_alg {

263

struct n2_hmac_alg {

264

const char *child_alg;

264

const char *child_alg;

265

struct n2_ahash_alg derived;

265

struct n2_ahash_alg derived;

266

};

266

};

267

268

static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)

268

static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)

269

{

269

{

270

struct crypto_alg *alg = tfm->__crt_alg;

270

struct crypto_alg *alg = tfm->__crt_alg;

271

struct ahash_alg *ahash_alg;

271

struct ahash_alg *ahash_alg;

272

273

ahash_alg = container_of(alg, struct ahash_alg, halg.base);

273

ahash_alg = container_of(alg, struct ahash_alg, halg.base);

274

275

return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);

275

return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);

276

}

276

}

277

278

struct n2_hash_ctx {

278

struct n2_hash_ctx {

279

struct crypto_ahash *fallback_tfm;

279

struct crypto_ahash *fallback_tfm;

280

};

280

};

281

282

#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */

282

#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */

283

284

struct n2_hmac_ctx {

284

struct n2_hmac_ctx {

285

struct n2_hash_ctx base;

285

struct n2_hash_ctx base;

286

287

struct crypto_shash *child_shash;

287

struct crypto_shash *child_shash;

288

289

int hash_key_len;

289

int hash_key_len;

290

unsigned char hash_key[N2_HASH_KEY_MAX];

290

unsigned char hash_key[N2_HASH_KEY_MAX];

291

};

291

};

292

293

struct n2_hash_req_ctx {

293

struct n2_hash_req_ctx {

294

union {

294

union {

295

struct md5_state md5;

295

struct md5_state md5;

296

struct sha1_state sha1;

296

struct sha1_state sha1;

297

struct sha256_state sha256;

297

struct sha256_state sha256;

298

} u;

298

} u;

299

300

struct ahash_request fallback_req;

300

struct ahash_request fallback_req;

301

};

301

};

302

303

static int n2_hash_async_init(struct ahash_request *req)

303

static int n2_hash_async_init(struct ahash_request *req)

304

{

304

{

305

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

305

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

306

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

306

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

307

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

307

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

308

309

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

309

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

310

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

310

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

311

312

return crypto_ahash_init(&rctx->fallback_req);

312

return crypto_ahash_init(&rctx->fallback_req);

313

}

313

}

314

315

static int n2_hash_async_update(struct ahash_request *req)

315

static int n2_hash_async_update(struct ahash_request *req)

316

{

316

{

317

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

317

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

318

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

318

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

319

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

319

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

320

321

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

321

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

322

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

322

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

323

rctx->fallback_req.nbytes = req->nbytes;

323

rctx->fallback_req.nbytes = req->nbytes;

324

rctx->fallback_req.src = req->src;

324

rctx->fallback_req.src = req->src;

325

326

return crypto_ahash_update(&rctx->fallback_req);

326

return crypto_ahash_update(&rctx->fallback_req);

327

}

327

}

328

329

static int n2_hash_async_final(struct ahash_request *req)

329

static int n2_hash_async_final(struct ahash_request *req)

330

{

330

{

331

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

331

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

332

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

332

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

333

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

333

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

334

335

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

335

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

336

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

336

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

337

rctx->fallback_req.result = req->result;

337

rctx->fallback_req.result = req->result;

338

339

return crypto_ahash_final(&rctx->fallback_req);

339

return crypto_ahash_final(&rctx->fallback_req);

340

}

340

}

341

342

static int n2_hash_async_finup(struct ahash_request *req)

342

static int n2_hash_async_finup(struct ahash_request *req)

343

{

343

{

344

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

344

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

345

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

345

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

346

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

346

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

347

348

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

348

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

349

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

349

rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

350

rctx->fallback_req.nbytes = req->nbytes;

350

rctx->fallback_req.nbytes = req->nbytes;

351

rctx->fallback_req.src = req->src;

351

rctx->fallback_req.src = req->src;

352

rctx->fallback_req.result = req->result;

352

rctx->fallback_req.result = req->result;

353

354

return crypto_ahash_finup(&rctx->fallback_req);

354

return crypto_ahash_finup(&rctx->fallback_req);

355

}

355

}

356

357

static int n2_hash_cra_init(struct crypto_tfm *tfm)

357

static int n2_hash_cra_init(struct crypto_tfm *tfm)

358

{

358

{

359

const char *fallback_driver_name = tfm->__crt_alg->cra_name;

359

const char *fallback_driver_name = tfm->__crt_alg->cra_name;

360

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

360

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

361

struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);

361

struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);

362

struct crypto_ahash *fallback_tfm;

362

struct crypto_ahash *fallback_tfm;

363

int err;

363

int err;

364

365

fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,

365

fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,

366

CRYPTO_ALG_NEED_FALLBACK);

366

CRYPTO_ALG_NEED_FALLBACK);

367

if (IS_ERR(fallback_tfm)) {

367

if (IS_ERR(fallback_tfm)) {

368

pr_warning("Fallback driver '%s' could not be loaded!\n",

368

pr_warning("Fallback driver '%s' could not be loaded!\n",

369

fallback_driver_name);

369

fallback_driver_name);

370

err = PTR_ERR(fallback_tfm);

370

err = PTR_ERR(fallback_tfm);

371

goto out;

371

goto out;

372

}

372

}

373

374

crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +

374

crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +

375

crypto_ahash_reqsize(fallback_tfm)));

375

crypto_ahash_reqsize(fallback_tfm)));

376

377

ctx->fallback_tfm = fallback_tfm;

377

ctx->fallback_tfm = fallback_tfm;

378

return 0;

378

return 0;

379

380

out:

380

out:

381

return err;

381

return err;

382

}

382

}

383

384

static void n2_hash_cra_exit(struct crypto_tfm *tfm)

384

static void n2_hash_cra_exit(struct crypto_tfm *tfm)

385

{

385

{

386

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

386

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

387

struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);

387

struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);

388

389

crypto_free_ahash(ctx->fallback_tfm);

389

crypto_free_ahash(ctx->fallback_tfm);

390

}

390

}

391

392

static int n2_hmac_cra_init(struct crypto_tfm *tfm)

392

static int n2_hmac_cra_init(struct crypto_tfm *tfm)

393

{

393

{

394

const char *fallback_driver_name = tfm->__crt_alg->cra_name;

394

const char *fallback_driver_name = tfm->__crt_alg->cra_name;

395

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

395

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

396

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);

396

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);

397

struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);

397

struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);

398

struct crypto_ahash *fallback_tfm;

398

struct crypto_ahash *fallback_tfm;

399

struct crypto_shash *child_shash;

399

struct crypto_shash *child_shash;

400

int err;

400

int err;

401

402

fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,

402

fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,

403

CRYPTO_ALG_NEED_FALLBACK);

403

CRYPTO_ALG_NEED_FALLBACK);

404

if (IS_ERR(fallback_tfm)) {

404

if (IS_ERR(fallback_tfm)) {

405

pr_warning("Fallback driver '%s' could not be loaded!\n",

405

pr_warning("Fallback driver '%s' could not be loaded!\n",

406

fallback_driver_name);

406

fallback_driver_name);

407

err = PTR_ERR(fallback_tfm);

407

err = PTR_ERR(fallback_tfm);

408

goto out;

408

goto out;

409

}

409

}

410

411

child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);

411

child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);

412

if (IS_ERR(child_shash)) {

412

if (IS_ERR(child_shash)) {

413

pr_warning("Child shash '%s' could not be loaded!\n",

413

pr_warning("Child shash '%s' could not be loaded!\n",

414

n2alg->child_alg);

414

n2alg->child_alg);

415

err = PTR_ERR(child_shash);

415

err = PTR_ERR(child_shash);

416

goto out_free_fallback;

416

goto out_free_fallback;

417

}

417

}

418

419

crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +

419

crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +

420

crypto_ahash_reqsize(fallback_tfm)));

420

crypto_ahash_reqsize(fallback_tfm)));

421

422

ctx->child_shash = child_shash;

422

ctx->child_shash = child_shash;

423

ctx->base.fallback_tfm = fallback_tfm;

423

ctx->base.fallback_tfm = fallback_tfm;

424

return 0;

424

return 0;

425

426

out_free_fallback:

426

out_free_fallback:

427

crypto_free_ahash(fallback_tfm);

427

crypto_free_ahash(fallback_tfm);

428

429

out:

429

out:

430

return err;

430

return err;

431

}

431

}

432

433

static void n2_hmac_cra_exit(struct crypto_tfm *tfm)

433

static void n2_hmac_cra_exit(struct crypto_tfm *tfm)

434

{

434

{

435

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

435

struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);

436

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);

436

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);

437

438

crypto_free_ahash(ctx->base.fallback_tfm);

438

crypto_free_ahash(ctx->base.fallback_tfm);

439

crypto_free_shash(ctx->child_shash);

439

crypto_free_shash(ctx->child_shash);

440

}

440

}

441

442

static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,

442

static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,

443

unsigned int keylen)

443

unsigned int keylen)

444

{

444

{

445

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);

445

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);

446

struct crypto_shash *child_shash = ctx->child_shash;

446

struct crypto_shash *child_shash = ctx->child_shash;

447

struct crypto_ahash *fallback_tfm;

447

struct crypto_ahash *fallback_tfm;

448

struct {

448

struct {

449

struct shash_desc shash;

449

struct shash_desc shash;

450

char ctx[crypto_shash_descsize(child_shash)];

450

char ctx[crypto_shash_descsize(child_shash)];

451

} desc;

451

} desc;

452

int err, bs, ds;

452

int err, bs, ds;

453

454

fallback_tfm = ctx->base.fallback_tfm;

454

fallback_tfm = ctx->base.fallback_tfm;

455

err = crypto_ahash_setkey(fallback_tfm, key, keylen);

455

err = crypto_ahash_setkey(fallback_tfm, key, keylen);

456

if (err)

456

if (err)

457

return err;

457

return err;

458

459

desc.shash.tfm = child_shash;

459

desc.shash.tfm = child_shash;

460

desc.shash.flags = crypto_ahash_get_flags(tfm) &

460

desc.shash.flags = crypto_ahash_get_flags(tfm) &

461

CRYPTO_TFM_REQ_MAY_SLEEP;

461

CRYPTO_TFM_REQ_MAY_SLEEP;

462

463

bs = crypto_shash_blocksize(child_shash);

463

bs = crypto_shash_blocksize(child_shash);

464

ds = crypto_shash_digestsize(child_shash);

464

ds = crypto_shash_digestsize(child_shash);

465

BUG_ON(ds > N2_HASH_KEY_MAX);

465

BUG_ON(ds > N2_HASH_KEY_MAX);

466

if (keylen > bs) {

466

if (keylen > bs) {

467

err = crypto_shash_digest(&desc.shash, key, keylen,

467

err = crypto_shash_digest(&desc.shash, key, keylen,

468

ctx->hash_key);

468

ctx->hash_key);

469

if (err)

469

if (err)

470

return err;

470

return err;

471

keylen = ds;

471

keylen = ds;

472

} else if (keylen <= N2_HASH_KEY_MAX)

472

} else if (keylen <= N2_HASH_KEY_MAX)

473

memcpy(ctx->hash_key, key, keylen);

473

memcpy(ctx->hash_key, key, keylen);

474

475

ctx->hash_key_len = keylen;

475

ctx->hash_key_len = keylen;

476

477

return err;

477

return err;

478

}

478

}

479

480

static unsigned long wait_for_tail(struct spu_queue *qp)

480

static unsigned long wait_for_tail(struct spu_queue *qp)

481

{

481

{

482

unsigned long head, hv_ret;

482

unsigned long head, hv_ret;

483

484

do {

484

do {

485

hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);

485

hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);

486

if (hv_ret != HV_EOK) {

486

if (hv_ret != HV_EOK) {

487

pr_err("Hypervisor error on gethead\n");

487

pr_err("Hypervisor error on gethead\n");

488

break;

488

break;

489

}

489

}

490

if (head == qp->tail) {

490

if (head == qp->tail) {

491

qp->head = head;

491

qp->head = head;

492

break;

492

break;

493

}

493

}

494

} while (1);

494

} while (1);

495

return hv_ret;

495

return hv_ret;

496

}

496

}

497

498

static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,

498

static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,

499

struct cwq_initial_entry *ent)

499

struct cwq_initial_entry *ent)

500

{

500

{

501

unsigned long hv_ret = spu_queue_submit(qp, ent);

501

unsigned long hv_ret = spu_queue_submit(qp, ent);

502

503

if (hv_ret == HV_EOK)

503

if (hv_ret == HV_EOK)

504

hv_ret = wait_for_tail(qp);

504

hv_ret = wait_for_tail(qp);

505

506

return hv_ret;

506

return hv_ret;

507

}

507

}

508

509

static int n2_do_async_digest(struct ahash_request *req,

509

static int n2_do_async_digest(struct ahash_request *req,

510

unsigned int auth_type, unsigned int digest_size,

510

unsigned int auth_type, unsigned int digest_size,

511

unsigned int result_size, void *hash_loc,

511

unsigned int result_size, void *hash_loc,

512

unsigned long auth_key, unsigned int auth_key_len)

512

unsigned long auth_key, unsigned int auth_key_len)

513

{

513

{

514

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

514

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

515

struct cwq_initial_entry *ent;

515

struct cwq_initial_entry *ent;

516

struct crypto_hash_walk walk;

516

struct crypto_hash_walk walk;

517

struct spu_queue *qp;

517

struct spu_queue *qp;

518

unsigned long flags;

518

unsigned long flags;

519

int err = -ENODEV;

519

int err = -ENODEV;

520

int nbytes, cpu;

520

int nbytes, cpu;

521

522

/* The total effective length of the operation may not

522

/* The total effective length of the operation may not

523

* exceed 2^16.

523

* exceed 2^16.

524

*/

524

*/

525

if (unlikely(req->nbytes > (1 << 16))) {

525

if (unlikely(req->nbytes > (1 << 16))) {

526

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

526

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

527

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

527

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

528

529

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

529

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

530

rctx->fallback_req.base.flags =

530

rctx->fallback_req.base.flags =

531

req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

531

req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

532

rctx->fallback_req.nbytes = req->nbytes;

532

rctx->fallback_req.nbytes = req->nbytes;

533

rctx->fallback_req.src = req->src;

533

rctx->fallback_req.src = req->src;

534

rctx->fallback_req.result = req->result;

534

rctx->fallback_req.result = req->result;

535

536

return crypto_ahash_digest(&rctx->fallback_req);

536

return crypto_ahash_digest(&rctx->fallback_req);

537

}

537

}

538

539

nbytes = crypto_hash_walk_first(req, &walk);

539

nbytes = crypto_hash_walk_first(req, &walk);

540

541

cpu = get_cpu();

541

cpu = get_cpu();

542

qp = cpu_to_cwq[cpu];

542

qp = cpu_to_cwq[cpu];

543

if (!qp)

543

if (!qp)

544

goto out;

544

goto out;

545

546

spin_lock_irqsave(&qp->lock, flags);

546

spin_lock_irqsave(&qp->lock, flags);

547

548

/* XXX can do better, improve this later by doing a by-hand scatterlist

548

/* XXX can do better, improve this later by doing a by-hand scatterlist

549

* XXX walk, etc.

549

* XXX walk, etc.

550

*/

550

*/

551

ent = qp->q + qp->tail;

551

ent = qp->q + qp->tail;

552

553

ent->control = control_word_base(nbytes, auth_key_len, 0,

553

ent->control = control_word_base(nbytes, auth_key_len, 0,

554

auth_type, digest_size,

554

auth_type, digest_size,

555

false, true, false, false,

555

false, true, false, false,

556

OPCODE_INPLACE_BIT |

556

OPCODE_INPLACE_BIT |

557

OPCODE_AUTH_MAC);

557

OPCODE_AUTH_MAC);

558

ent->src_addr = __pa(walk.data);

558

ent->src_addr = __pa(walk.data);

559

ent->auth_key_addr = auth_key;

559

ent->auth_key_addr = auth_key;

560

ent->auth_iv_addr = __pa(hash_loc);

560

ent->auth_iv_addr = __pa(hash_loc);

561

ent->final_auth_state_addr = 0UL;

561

ent->final_auth_state_addr = 0UL;

562

ent->enc_key_addr = 0UL;

562

ent->enc_key_addr = 0UL;

563

ent->enc_iv_addr = 0UL;

563

ent->enc_iv_addr = 0UL;

564

ent->dest_addr = __pa(hash_loc);

564

ent->dest_addr = __pa(hash_loc);

565

566

nbytes = crypto_hash_walk_done(&walk, 0);

566

nbytes = crypto_hash_walk_done(&walk, 0);

567

while (nbytes > 0) {

567

while (nbytes > 0) {

568

ent = spu_queue_next(qp, ent);

568

ent = spu_queue_next(qp, ent);

569

570

ent->control = (nbytes - 1);

570

ent->control = (nbytes - 1);

571

ent->src_addr = __pa(walk.data);

571

ent->src_addr = __pa(walk.data);

572

ent->auth_key_addr = 0UL;

572

ent->auth_key_addr = 0UL;

573

ent->auth_iv_addr = 0UL;

573

ent->auth_iv_addr = 0UL;

574

ent->final_auth_state_addr = 0UL;

574

ent->final_auth_state_addr = 0UL;

575

ent->enc_key_addr = 0UL;

575

ent->enc_key_addr = 0UL;

576

ent->enc_iv_addr = 0UL;

576

ent->enc_iv_addr = 0UL;

577

ent->dest_addr = 0UL;

577

ent->dest_addr = 0UL;

578

579

nbytes = crypto_hash_walk_done(&walk, 0);

579

nbytes = crypto_hash_walk_done(&walk, 0);

580

}

580

}

581

ent->control |= CONTROL_END_OF_BLOCK;

581

ent->control |= CONTROL_END_OF_BLOCK;

582

583

if (submit_and_wait_for_tail(qp, ent) != HV_EOK)

583

if (submit_and_wait_for_tail(qp, ent) != HV_EOK)

584

err = -EINVAL;

584

err = -EINVAL;

585

else

585

else

586

err = 0;

586

err = 0;

587

588

spin_unlock_irqrestore(&qp->lock, flags);

588

spin_unlock_irqrestore(&qp->lock, flags);

589

590

if (!err)

590

if (!err)

591

memcpy(req->result, hash_loc, result_size);

591

memcpy(req->result, hash_loc, result_size);

592

out:

592

out:

593

put_cpu();

593

put_cpu();

594

595

return err;

595

return err;

596

}

596

}

597

598

static int n2_hash_async_digest(struct ahash_request *req)

598

static int n2_hash_async_digest(struct ahash_request *req)

599

{

599

{

600

struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);

600

struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);

601

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

601

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

602

int ds;

602

int ds;

603

604

ds = n2alg->digest_size;

604

ds = n2alg->digest_size;

605

if (unlikely(req->nbytes == 0)) {

605

if (unlikely(req->nbytes == 0)) {

606

memcpy(req->result, n2alg->hash_zero, ds);

606

memcpy(req->result, n2alg->hash_zero, ds);

607

return 0;

607

return 0;

608

}

608

}

609

memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);

609

memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);

610

611

return n2_do_async_digest(req, n2alg->auth_type,

611

return n2_do_async_digest(req, n2alg->auth_type,

612

n2alg->hw_op_hashsz, ds,

612

n2alg->hw_op_hashsz, ds,

613

&rctx->u, 0UL, 0);

613

&rctx->u, 0UL, 0);

614

}

614

}

615

616

static int n2_hmac_async_digest(struct ahash_request *req)

616

static int n2_hmac_async_digest(struct ahash_request *req)

617

{

617

{

618

struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);

618

struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);

619

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

619

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

620

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

620

struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

621

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);

621

struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);

622

int ds;

622

int ds;

623

624

ds = n2alg->derived.digest_size;

624

ds = n2alg->derived.digest_size;

625

if (unlikely(req->nbytes == 0) ||

625

if (unlikely(req->nbytes == 0) ||

626

unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {

626

unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {

627

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

627

struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);

628

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

628

struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);

629

630

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

630

ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);

631

rctx->fallback_req.base.flags =

631

rctx->fallback_req.base.flags =

632

req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

632

req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;

633

rctx->fallback_req.nbytes = req->nbytes;

633

rctx->fallback_req.nbytes = req->nbytes;

634

rctx->fallback_req.src = req->src;

634

rctx->fallback_req.src = req->src;

635

rctx->fallback_req.result = req->result;

635

rctx->fallback_req.result = req->result;

636

637

return crypto_ahash_digest(&rctx->fallback_req);

637

return crypto_ahash_digest(&rctx->fallback_req);

638

}

638

}

639

memcpy(&rctx->u, n2alg->derived.hash_init,

639

memcpy(&rctx->u, n2alg->derived.hash_init,

640

n2alg->derived.hw_op_hashsz);

640

n2alg->derived.hw_op_hashsz);

641

642

return n2_do_async_digest(req, n2alg->derived.hmac_type,

642

return n2_do_async_digest(req, n2alg->derived.hmac_type,

643

n2alg->derived.hw_op_hashsz, ds,

643

n2alg->derived.hw_op_hashsz, ds,

644

&rctx->u,

644

&rctx->u,

645

__pa(&ctx->hash_key),

645

__pa(&ctx->hash_key),

646

ctx->hash_key_len);

646

ctx->hash_key_len);

647

}

647

}

648

649

struct n2_cipher_context {

649

struct n2_cipher_context {

650

int key_len;

650

int key_len;

651

int enc_type;

651

int enc_type;

652

union {

652

union {

653

u8 aes[AES_MAX_KEY_SIZE];

653

u8 aes[AES_MAX_KEY_SIZE];

654

u8 des[DES_KEY_SIZE];

654

u8 des[DES_KEY_SIZE];

655

u8 des3[3 * DES_KEY_SIZE];

655

u8 des3[3 * DES_KEY_SIZE];

656

u8 arc4[258]; /* S-box, X, Y */

656

u8 arc4[258]; /* S-box, X, Y */

657

} key;

657

} key;

658

};

658

};

659

660

#define N2_CHUNK_ARR_LEN 16

660

#define N2_CHUNK_ARR_LEN 16

661

662

struct n2_crypto_chunk {

662

struct n2_crypto_chunk {

663

struct list_head entry;

663

struct list_head entry;

664

unsigned long iv_paddr : 44;

664

unsigned long iv_paddr : 44;

665

unsigned long arr_len : 20;

665

unsigned long arr_len : 20;

666

unsigned long dest_paddr;

666

unsigned long dest_paddr;

667

unsigned long dest_final;

667

unsigned long dest_final;

668

struct {

668

struct {

669

unsigned long src_paddr : 44;

669

unsigned long src_paddr : 44;

670

unsigned long src_len : 20;

670

unsigned long src_len : 20;

671

} arr[N2_CHUNK_ARR_LEN];

671

} arr[N2_CHUNK_ARR_LEN];

672

};

672

};

673

674

struct n2_request_context {

674

struct n2_request_context {

675

struct ablkcipher_walk walk;

675

struct ablkcipher_walk walk;

676

struct list_head chunk_list;

676

struct list_head chunk_list;

677

struct n2_crypto_chunk chunk;

677

struct n2_crypto_chunk chunk;

678

u8 temp_iv[16];

678

u8 temp_iv[16];

679

};

679

};

680

681

/* The SPU allows some level of flexibility for partial cipher blocks

681

/* The SPU allows some level of flexibility for partial cipher blocks

682

* being specified in a descriptor.

682

* being specified in a descriptor.

683

*

683

*

684

* It merely requires that every descriptor's length field is at least

684

* It merely requires that every descriptor's length field is at least

685

* as large as the cipher block size. This means that a cipher block

685

* as large as the cipher block size. This means that a cipher block

686

* can span at most 2 descriptors. However, this does not allow a

686

* can span at most 2 descriptors. However, this does not allow a

687

* partial block to span into the final descriptor as that would

687

* partial block to span into the final descriptor as that would

688

* violate the rule (since every descriptor's length must be at lest

688

* violate the rule (since every descriptor's length must be at lest

689

* the block size). So, for example, assuming an 8 byte block size:

689

* the block size). So, for example, assuming an 8 byte block size:

690

*

690

*

691

* 0xe --> 0xa --> 0x8

691

* 0xe --> 0xa --> 0x8

692

*

692

*

693

* is a valid length sequence, whereas:

693

* is a valid length sequence, whereas:

694

*

694

*

695

* 0xe --> 0xb --> 0x7

695

* 0xe --> 0xb --> 0x7

696

*

696

*

697

* is not a valid sequence.

697

* is not a valid sequence.

698

*/

698

*/

699

700

struct n2_cipher_alg {

700

struct n2_cipher_alg {

701

struct list_head entry;

701

struct list_head entry;

702

u8 enc_type;

702

u8 enc_type;

703

struct crypto_alg alg;

703

struct crypto_alg alg;

704

};

704

};

705

706

static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)

706

static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)

707

{

707

{

708

struct crypto_alg *alg = tfm->__crt_alg;

708

struct crypto_alg *alg = tfm->__crt_alg;

709

710

return container_of(alg, struct n2_cipher_alg, alg);

710

return container_of(alg, struct n2_cipher_alg, alg);

711

}

711

}

712

713

struct n2_cipher_request_context {

713

struct n2_cipher_request_context {

714

struct ablkcipher_walk walk;

714

struct ablkcipher_walk walk;

715

};

715

};

716

717

static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

717

static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

718

unsigned int keylen)

718

unsigned int keylen)

719

{

719

{

720

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

720

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

721

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

721

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

722

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

722

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

723

724

ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);

724

ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);

725

726

switch (keylen) {

726

switch (keylen) {

727

case AES_KEYSIZE_128:

727

case AES_KEYSIZE_128:

728

ctx->enc_type |= ENC_TYPE_ALG_AES128;

728

ctx->enc_type |= ENC_TYPE_ALG_AES128;

729

break;

729

break;

730

case AES_KEYSIZE_192:

730

case AES_KEYSIZE_192:

731

ctx->enc_type |= ENC_TYPE_ALG_AES192;

731

ctx->enc_type |= ENC_TYPE_ALG_AES192;

732

break;

732

break;

733

case AES_KEYSIZE_256:

733

case AES_KEYSIZE_256:

734

ctx->enc_type |= ENC_TYPE_ALG_AES256;

734

ctx->enc_type |= ENC_TYPE_ALG_AES256;

735

break;

735

break;

736

default:

736

default:

737

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

737

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

738

return -EINVAL;

738

return -EINVAL;

739

}

739

}

740

741

ctx->key_len = keylen;

741

ctx->key_len = keylen;

742

memcpy(ctx->key.aes, key, keylen);

742

memcpy(ctx->key.aes, key, keylen);

743

return 0;

743

return 0;

744

}

744

}

745

746

static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

746

static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

747

unsigned int keylen)

747

unsigned int keylen)

748

{

748

{

749

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

749

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

750

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

750

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

751

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

751

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

752

u32 tmp[DES_EXPKEY_WORDS];

752

u32 tmp[DES_EXPKEY_WORDS];

753

int err;

753

int err;

754

755

ctx->enc_type = n2alg->enc_type;

755

ctx->enc_type = n2alg->enc_type;

756

757

if (keylen != DES_KEY_SIZE) {

757

if (keylen != DES_KEY_SIZE) {

758

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

758

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

759

return -EINVAL;

759

return -EINVAL;

760

}

760

}

761

762

err = des_ekey(tmp, key);

762

err = des_ekey(tmp, key);

763

if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {

763

if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {

764

tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;

764

tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;

765

return -EINVAL;

765

return -EINVAL;

766

}

766

}

767

768

ctx->key_len = keylen;

768

ctx->key_len = keylen;

769

memcpy(ctx->key.des, key, keylen);

769

memcpy(ctx->key.des, key, keylen);

770

return 0;

770

return 0;

771

}

771

}

772

773

static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

773

static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

774

unsigned int keylen)

774

unsigned int keylen)

775

{

775

{

776

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

776

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

777

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

777

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

778

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

778

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

779

780

ctx->enc_type = n2alg->enc_type;

780

ctx->enc_type = n2alg->enc_type;

781

782

if (keylen != (3 * DES_KEY_SIZE)) {

782

if (keylen != (3 * DES_KEY_SIZE)) {

783

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

783

crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);

784

return -EINVAL;

784

return -EINVAL;

785

}

785

}

786

ctx->key_len = keylen;

786

ctx->key_len = keylen;

787

memcpy(ctx->key.des3, key, keylen);

787

memcpy(ctx->key.des3, key, keylen);

788

return 0;

788

return 0;

789

}

789

}

790

791

static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

791

static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,

792

unsigned int keylen)

792

unsigned int keylen)

793

{

793

{

794

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

794

struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);

795

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

795

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

796

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

796

struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);

797

u8 *s = ctx->key.arc4;

797

u8 *s = ctx->key.arc4;

798

u8 *x = s + 256;

798

u8 *x = s + 256;

799

u8 *y = x + 1;

799

u8 *y = x + 1;

800

int i, j, k;

800

int i, j, k;

801

802

ctx->enc_type = n2alg->enc_type;

802

ctx->enc_type = n2alg->enc_type;

803

804

j = k = 0;

804

j = k = 0;

805

*x = 0;

805

*x = 0;

806

*y = 0;

806

*y = 0;

807

for (i = 0; i < 256; i++)

807

for (i = 0; i < 256; i++)

808

s[i] = i;

808

s[i] = i;

809

for (i = 0; i < 256; i++) {

809

for (i = 0; i < 256; i++) {

810

u8 a = s[i];

810

u8 a = s[i];

811

j = (j + key[k] + a) & 0xff;

811

j = (j + key[k] + a) & 0xff;

812

s[i] = s[j];

812

s[i] = s[j];

813

s[j] = a;

813

s[j] = a;

814

if (++k >= keylen)

814

if (++k >= keylen)

815

k = 0;

815

k = 0;

816

}

816

}

817

818

return 0;

818

return 0;

819

}

819

}

820

821

static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)

821

static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)

822

{

822

{

823

int this_len = nbytes;

823

int this_len = nbytes;

824

825

this_len -= (nbytes & (block_size - 1));

825

this_len -= (nbytes & (block_size - 1));

826

return this_len > (1 << 16) ? (1 << 16) : this_len;

826

return this_len > (1 << 16) ? (1 << 16) : this_len;

827

}

827

}

828

829

static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,

829

static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,

830

struct spu_queue *qp, bool encrypt)

830

struct spu_queue *qp, bool encrypt)

831

{

831

{

832

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

832

struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);

833

struct cwq_initial_entry *ent;

833

struct cwq_initial_entry *ent;

834

bool in_place;

834

bool in_place;

835

int i;

835

int i;

836

837

ent = spu_queue_alloc(qp, cp->arr_len);

837

ent = spu_queue_alloc(qp, cp->arr_len);

838

if (!ent) {

838

if (!ent) {

839

pr_info("queue_alloc() of %d fails\n",

839

pr_info("queue_alloc() of %d fails\n",

840

cp->arr_len);

840

cp->arr_len);

841

return -EBUSY;

841

return -EBUSY;

842

}

842

}

843

844

in_place = (cp->dest_paddr == cp->arr[0].src_paddr);

844

in_place = (cp->dest_paddr == cp->arr[0].src_paddr);

845

846

ent->control = control_word_base(cp->arr[0].src_len,

846

ent->control = control_word_base(cp->arr[0].src_len,

847

0, ctx->enc_type, 0, 0,

847

0, ctx->enc_type, 0, 0,

848

false, true, false, encrypt,

848

false, true, false, encrypt,

849

OPCODE_ENCRYPT |

849

OPCODE_ENCRYPT |

850

(in_place ? OPCODE_INPLACE_BIT : 0));

850

(in_place ? OPCODE_INPLACE_BIT : 0));

851

ent->src_addr = cp->arr[0].src_paddr;

851

ent->src_addr = cp->arr[0].src_paddr;

852

ent->auth_key_addr = 0UL;

852

ent->auth_key_addr = 0UL;

853

ent->auth_iv_addr = 0UL;

853

ent->auth_iv_addr = 0UL;

854

ent->final_auth_state_addr = 0UL;

854

ent->final_auth_state_addr = 0UL;

855

ent->enc_key_addr = __pa(&ctx->key);

855

ent->enc_key_addr = __pa(&ctx->key);

856

ent->enc_iv_addr = cp->iv_paddr;

856

ent->enc_iv_addr = cp->iv_paddr;

857

ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);

857

ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);

858

859

for (i = 1; i < cp->arr_len; i++) {

859

for (i = 1; i < cp->arr_len; i++) {

860

ent = spu_queue_next(qp, ent);

860

ent = spu_queue_next(qp, ent);

861

862

ent->control = cp->arr[i].src_len - 1;

862

ent->control = cp->arr[i].src_len - 1;

863

ent->src_addr = cp->arr[i].src_paddr;

863

ent->src_addr = cp->arr[i].src_paddr;

864

ent->auth_key_addr = 0UL;

864

ent->auth_key_addr = 0UL;

865

ent->auth_iv_addr = 0UL;

865

ent->auth_iv_addr = 0UL;

866

ent->final_auth_state_addr = 0UL;

866

ent->final_auth_state_addr = 0UL;

867

ent->enc_key_addr = 0UL;

867

ent->enc_key_addr = 0UL;

868

ent->enc_iv_addr = 0UL;

868

ent->enc_iv_addr = 0UL;

869

ent->dest_addr = 0UL;

869

ent->dest_addr = 0UL;

870

}

870

}

871

ent->control |= CONTROL_END_OF_BLOCK;

871

ent->control |= CONTROL_END_OF_BLOCK;

872

873

return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;

873

return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;

874

}

874

}

875

876

static int n2_compute_chunks(struct ablkcipher_request *req)

876

static int n2_compute_chunks(struct ablkcipher_request *req)

877

{

877

{

878

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

878

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

879

struct ablkcipher_walk *walk = &rctx->walk;

879

struct ablkcipher_walk *walk = &rctx->walk;

880

struct n2_crypto_chunk *chunk;

880

struct n2_crypto_chunk *chunk;

881

unsigned long dest_prev;

881

unsigned long dest_prev;

882

unsigned int tot_len;

882

unsigned int tot_len;

883

bool prev_in_place;

883

bool prev_in_place;

884

int err, nbytes;

884

int err, nbytes;

885

886

ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);

886

ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);

887

err = ablkcipher_walk_phys(req, walk);

887

err = ablkcipher_walk_phys(req, walk);

888

if (err)

888

if (err)

889

return err;

889

return err;

890

891

INIT_LIST_HEAD(&rctx->chunk_list);

891

INIT_LIST_HEAD(&rctx->chunk_list);

892

893

chunk = &rctx->chunk;

893

chunk = &rctx->chunk;

894

INIT_LIST_HEAD(&chunk->entry);

894

INIT_LIST_HEAD(&chunk->entry);

895

896

chunk->iv_paddr = 0UL;

896

chunk->iv_paddr = 0UL;

897

chunk->arr_len = 0;

897

chunk->arr_len = 0;

898

chunk->dest_paddr = 0UL;

898

chunk->dest_paddr = 0UL;

899

900

prev_in_place = false;

900

prev_in_place = false;

901

dest_prev = ~0UL;

901

dest_prev = ~0UL;

902

tot_len = 0;

902

tot_len = 0;

903

904

while ((nbytes = walk->nbytes) != 0) {

904

while ((nbytes = walk->nbytes) != 0) {

905

unsigned long dest_paddr, src_paddr;

905

unsigned long dest_paddr, src_paddr;

906

bool in_place;

906

bool in_place;

907

int this_len;

907

int this_len;

908

909

src_paddr = (page_to_phys(walk->src.page) +

909

src_paddr = (page_to_phys(walk->src.page) +

910

walk->src.offset);

910

walk->src.offset);

911

dest_paddr = (page_to_phys(walk->dst.page) +

911

dest_paddr = (page_to_phys(walk->dst.page) +

912

walk->dst.offset);

912

walk->dst.offset);

913

in_place = (src_paddr == dest_paddr);

913

in_place = (src_paddr == dest_paddr);

914

this_len = cipher_descriptor_len(nbytes, walk->blocksize);

914

this_len = cipher_descriptor_len(nbytes, walk->blocksize);

915

916

if (chunk->arr_len != 0) {

916

if (chunk->arr_len != 0) {

917

if (in_place != prev_in_place ||

917

if (in_place != prev_in_place ||

918

(!prev_in_place &&

918

(!prev_in_place &&

919

dest_paddr != dest_prev) ||

919

dest_paddr != dest_prev) ||

920

chunk->arr_len == N2_CHUNK_ARR_LEN ||

920

chunk->arr_len == N2_CHUNK_ARR_LEN ||

921

tot_len + this_len > (1 << 16)) {

921

tot_len + this_len > (1 << 16)) {

922

chunk->dest_final = dest_prev;

922

chunk->dest_final = dest_prev;

923

list_add_tail(&chunk->entry,

923

list_add_tail(&chunk->entry,

924

&rctx->chunk_list);

924

&rctx->chunk_list);

925

chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);

925

chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);

926

if (!chunk) {

926

if (!chunk) {

927

err = -ENOMEM;

927

err = -ENOMEM;

928

break;

928

break;

929

}

929

}

930

INIT_LIST_HEAD(&chunk->entry);

930

INIT_LIST_HEAD(&chunk->entry);

931

}

931

}

932

}

932

}

933

if (chunk->arr_len == 0) {

933

if (chunk->arr_len == 0) {

934

chunk->dest_paddr = dest_paddr;

934

chunk->dest_paddr = dest_paddr;

935

tot_len = 0;

935

tot_len = 0;

936

}

936

}

937

chunk->arr[chunk->arr_len].src_paddr = src_paddr;

937

chunk->arr[chunk->arr_len].src_paddr = src_paddr;

938

chunk->arr[chunk->arr_len].src_len = this_len;

938

chunk->arr[chunk->arr_len].src_len = this_len;

939

chunk->arr_len++;

939

chunk->arr_len++;

940

941

dest_prev = dest_paddr + this_len;

941

dest_prev = dest_paddr + this_len;

942

prev_in_place = in_place;

942

prev_in_place = in_place;

943

tot_len += this_len;

943

tot_len += this_len;

944

945

err = ablkcipher_walk_done(req, walk, nbytes - this_len);

945

err = ablkcipher_walk_done(req, walk, nbytes - this_len);

946

if (err)

946

if (err)

947

break;

947

break;

948

}

948

}

949

if (!err && chunk->arr_len != 0) {

949

if (!err && chunk->arr_len != 0) {

950

chunk->dest_final = dest_prev;

950

chunk->dest_final = dest_prev;

951

list_add_tail(&chunk->entry, &rctx->chunk_list);

951

list_add_tail(&chunk->entry, &rctx->chunk_list);

952

}

952

}

953

954

return err;

954

return err;

955

}

955

}

956

957

static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)

957

static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)

958

{

958

{

959

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

959

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

960

struct n2_crypto_chunk *c, *tmp;

960

struct n2_crypto_chunk *c, *tmp;

961

962

if (final_iv)

962

if (final_iv)

963

memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);

963

memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);

964

965

ablkcipher_walk_complete(&rctx->walk);

965

ablkcipher_walk_complete(&rctx->walk);

966

list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {

966

list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {

967

list_del(&c->entry);

967

list_del(&c->entry);

968

if (unlikely(c != &rctx->chunk))

968

if (unlikely(c != &rctx->chunk))

969

kfree(c);

969

kfree(c);

970

}

970

}

971

972

}

972

}

973

974

static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)

974

static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)

975

{

975

{

976

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

976

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

977

struct crypto_tfm *tfm = req->base.tfm;

977

struct crypto_tfm *tfm = req->base.tfm;

978

int err = n2_compute_chunks(req);

978

int err = n2_compute_chunks(req);

979

struct n2_crypto_chunk *c, *tmp;

979

struct n2_crypto_chunk *c, *tmp;

980

unsigned long flags, hv_ret;

980

unsigned long flags, hv_ret;

981

struct spu_queue *qp;

981

struct spu_queue *qp;

982

983

if (err)

983

if (err)

984

return err;

984

return err;

985

986

qp = cpu_to_cwq[get_cpu()];

986

qp = cpu_to_cwq[get_cpu()];

987

err = -ENODEV;

987

err = -ENODEV;

988

if (!qp)

988

if (!qp)

989

goto out;

989

goto out;

990

991

spin_lock_irqsave(&qp->lock, flags);

991

spin_lock_irqsave(&qp->lock, flags);

992

993

list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {

993

list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {

994

err = __n2_crypt_chunk(tfm, c, qp, encrypt);

994

err = __n2_crypt_chunk(tfm, c, qp, encrypt);

995

if (err)

995

if (err)

996

break;

996

break;

997

list_del(&c->entry);

997

list_del(&c->entry);

998

if (unlikely(c != &rctx->chunk))

998

if (unlikely(c != &rctx->chunk))

999

kfree(c);

999

kfree(c);

1000

}

1000

}

1001

if (!err) {

1001

if (!err) {

1002

hv_ret = wait_for_tail(qp);

1002

hv_ret = wait_for_tail(qp);

1003

if (hv_ret != HV_EOK)

1003

if (hv_ret != HV_EOK)

1004

err = -EINVAL;

1004

err = -EINVAL;

1005

}

1005

}

1006

1007

spin_unlock_irqrestore(&qp->lock, flags);

1007

spin_unlock_irqrestore(&qp->lock, flags);

1008

1009

out:

1009

put_cpu();

1010

put_cpu();

1010

1011

out:

1012

n2_chunk_complete(req, NULL);

1012

n2_chunk_complete(req, NULL);

1013

return err;

1013

return err;

1014

}

1014

}

1015

1016

static int n2_encrypt_ecb(struct ablkcipher_request *req)

1016

static int n2_encrypt_ecb(struct ablkcipher_request *req)

1017

{

1017

{

1018

return n2_do_ecb(req, true);

1018

return n2_do_ecb(req, true);

1019

}

1019

}

1020

1021

static int n2_decrypt_ecb(struct ablkcipher_request *req)

1021

static int n2_decrypt_ecb(struct ablkcipher_request *req)

1022

{

1022

{

1023

return n2_do_ecb(req, false);

1023

return n2_do_ecb(req, false);

1024

}

1024

}

1025

1026

static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)

1026

static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)

1027

{

1027

{

1028

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

1028

struct n2_request_context *rctx = ablkcipher_request_ctx(req);

1029

struct crypto_tfm *tfm = req->base.tfm;

1029

struct crypto_tfm *tfm = req->base.tfm;

1030

unsigned long flags, hv_ret, iv_paddr;

1030

unsigned long flags, hv_ret, iv_paddr;

1031

int err = n2_compute_chunks(req);

1031

int err = n2_compute_chunks(req);

1032

struct n2_crypto_chunk *c, *tmp;

1032

struct n2_crypto_chunk *c, *tmp;

1033

struct spu_queue *qp;

1033

struct spu_queue *qp;

1034

void *final_iv_addr;

1034

void *final_iv_addr;

1035

1036

final_iv_addr = NULL;

1036

final_iv_addr = NULL;

1037

1038

if (err)

1038

if (err)

1039

return err;

1039

return err;

1040

1041

qp = cpu_to_cwq[get_cpu()];

1041

qp = cpu_to_cwq[get_cpu()];

1042

err = -ENODEV;

1042

err = -ENODEV;

1043

if (!qp)

1043

if (!qp)

1044

goto out;

1044

goto out;

1045

1046

spin_lock_irqsave(&qp->lock, flags);

1046

spin_lock_irqsave(&qp->lock, flags);

1047

1048

if (encrypt) {

1048

if (encrypt) {

1049

iv_paddr = __pa(rctx->walk.iv);

1049

iv_paddr = __pa(rctx->walk.iv);

1050

list_for_each_entry_safe(c, tmp, &rctx->chunk_list,

1050

list_for_each_entry_safe(c, tmp, &rctx->chunk_list,

1051

entry) {

1051

entry) {

1052

c->iv_paddr = iv_paddr;

1052

c->iv_paddr = iv_paddr;

1053

err = __n2_crypt_chunk(tfm, c, qp, true);

1053

err = __n2_crypt_chunk(tfm, c, qp, true);

1054

if (err)

1054

if (err)

1055

break;

1055

break;

1056

iv_paddr = c->dest_final - rctx->walk.blocksize;

1056

iv_paddr = c->dest_final - rctx->walk.blocksize;

1057

list_del(&c->entry);

1057

list_del(&c->entry);

1058

if (unlikely(c != &rctx->chunk))

1058

if (unlikely(c != &rctx->chunk))

1059

kfree(c);

1059

kfree(c);

1060

}

1060

}

1061

final_iv_addr = __va(iv_paddr);

1061

final_iv_addr = __va(iv_paddr);

1062

} else {

1062

} else {

1063

list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,

1063

list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,

1064

entry) {

1064

entry) {

1065

if (c == &rctx->chunk) {

1065

if (c == &rctx->chunk) {

1066

iv_paddr = __pa(rctx->walk.iv);

1066

iv_paddr = __pa(rctx->walk.iv);

1067

} else {

1067

} else {

1068

iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +

1068

iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +

1069

tmp->arr[tmp->arr_len-1].src_len -

1069

tmp->arr[tmp->arr_len-1].src_len -

1070

rctx->walk.blocksize);

1070

rctx->walk.blocksize);

1071

}

1071

}

1072

if (!final_iv_addr) {

1072

if (!final_iv_addr) {

1073

unsigned long pa;

1073

unsigned long pa;

1074

1075

pa = (c->arr[c->arr_len-1].src_paddr +

1075

pa = (c->arr[c->arr_len-1].src_paddr +

1076

c->arr[c->arr_len-1].src_len -

1076

c->arr[c->arr_len-1].src_len -

1077

rctx->walk.blocksize);

1077

rctx->walk.blocksize);

1078

final_iv_addr = rctx->temp_iv;

1078

final_iv_addr = rctx->temp_iv;

1079

memcpy(rctx->temp_iv, __va(pa),

1079

memcpy(rctx->temp_iv, __va(pa),

1080

rctx->walk.blocksize);

1080

rctx->walk.blocksize);

1081

}

1081

}

1082

c->iv_paddr = iv_paddr;

1082

c->iv_paddr = iv_paddr;

1083

err = __n2_crypt_chunk(tfm, c, qp, false);

1083

err = __n2_crypt_chunk(tfm, c, qp, false);

1084

if (err)

1084

if (err)

1085

break;

1085

break;

1086

list_del(&c->entry);

1086

list_del(&c->entry);

1087

if (unlikely(c != &rctx->chunk))

1087

if (unlikely(c != &rctx->chunk))

1088

kfree(c);

1088

kfree(c);

1089

}

1089

}

1090

}

1090

}

1091

if (!err) {

1091

if (!err) {

1092

hv_ret = wait_for_tail(qp);

1092

hv_ret = wait_for_tail(qp);

1093

if (hv_ret != HV_EOK)

1093

if (hv_ret != HV_EOK)

1094

err = -EINVAL;

1094

err = -EINVAL;

1095

}

1095

}

1096

1097

spin_unlock_irqrestore(&qp->lock, flags);

1097

spin_unlock_irqrestore(&qp->lock, flags);

1098

1099

out:

1099

put_cpu();

1100

put_cpu();

1100

1101

out:

1102

n2_chunk_complete(req, err ? NULL : final_iv_addr);

1102

n2_chunk_complete(req, err ? NULL : final_iv_addr);

1103

return err;

1103

return err;

1104

}

1104

}

1105

1106

static int n2_encrypt_chaining(struct ablkcipher_request *req)

1106

static int n2_encrypt_chaining(struct ablkcipher_request *req)

1107

{

1107

{

1108

return n2_do_chaining(req, true);

1108

return n2_do_chaining(req, true);

1109

}

1109

}

1110

1111

static int n2_decrypt_chaining(struct ablkcipher_request *req)

1111

static int n2_decrypt_chaining(struct ablkcipher_request *req)

1112

{

1112

{

1113

return n2_do_chaining(req, false);

1113

return n2_do_chaining(req, false);

1114

}

1114

}

1115

1116

struct n2_cipher_tmpl {

1116

struct n2_cipher_tmpl {

1117

const char *name;

1117

const char *name;

1118

const char *drv_name;

1118

const char *drv_name;

1119

u8 block_size;

1119

u8 block_size;

1120

u8 enc_type;

1120

u8 enc_type;

1121

struct ablkcipher_alg ablkcipher;

1121

struct ablkcipher_alg ablkcipher;

1122

};

1122

};

1123

1124

static const struct n2_cipher_tmpl cipher_tmpls[] = {

1124

static const struct n2_cipher_tmpl cipher_tmpls[] = {

1125

/* ARC4: only ECB is supported (chaining bits ignored) */

1125

/* ARC4: only ECB is supported (chaining bits ignored) */

1126

{ .name = "ecb(arc4)",

1126

{ .name = "ecb(arc4)",

1127

.drv_name = "ecb-arc4",

1127

.drv_name = "ecb-arc4",

1128

.block_size = 1,

1128

.block_size = 1,

1129

.enc_type = (ENC_TYPE_ALG_RC4_STREAM |

1129

.enc_type = (ENC_TYPE_ALG_RC4_STREAM |

1130

ENC_TYPE_CHAINING_ECB),

1130

ENC_TYPE_CHAINING_ECB),

1131

.ablkcipher = {

1131

.ablkcipher = {

1132

.min_keysize = 1,

1132

.min_keysize = 1,

1133

.max_keysize = 256,

1133

.max_keysize = 256,

1134

.setkey = n2_arc4_setkey,

1134

.setkey = n2_arc4_setkey,

1135

.encrypt = n2_encrypt_ecb,

1135

.encrypt = n2_encrypt_ecb,

1136

.decrypt = n2_decrypt_ecb,

1136

.decrypt = n2_decrypt_ecb,

1137

},

1137

},

1138

},

1138

},

1139

1140

/* DES: ECB CBC and CFB are supported */

1140

/* DES: ECB CBC and CFB are supported */

1141

{ .name = "ecb(des)",

1141

{ .name = "ecb(des)",

1142

.drv_name = "ecb-des",

1142

.drv_name = "ecb-des",

1143

.block_size = DES_BLOCK_SIZE,

1143

.block_size = DES_BLOCK_SIZE,

1144

.enc_type = (ENC_TYPE_ALG_DES |

1144

.enc_type = (ENC_TYPE_ALG_DES |

1145

ENC_TYPE_CHAINING_ECB),

1145

ENC_TYPE_CHAINING_ECB),

1146

.ablkcipher = {

1146

.ablkcipher = {

1147

.min_keysize = DES_KEY_SIZE,

1147

.min_keysize = DES_KEY_SIZE,

1148

.max_keysize = DES_KEY_SIZE,

1148

.max_keysize = DES_KEY_SIZE,

1149

.setkey = n2_des_setkey,

1149

.setkey = n2_des_setkey,

1150

.encrypt = n2_encrypt_ecb,

1150

.encrypt = n2_encrypt_ecb,

1151

.decrypt = n2_decrypt_ecb,

1151

.decrypt = n2_decrypt_ecb,

1152

},

1152

},

1153

},

1153

},

1154

{ .name = "cbc(des)",

1154

{ .name = "cbc(des)",

1155

.drv_name = "cbc-des",

1155

.drv_name = "cbc-des",

1156

.block_size = DES_BLOCK_SIZE,

1156

.block_size = DES_BLOCK_SIZE,

1157

.enc_type = (ENC_TYPE_ALG_DES |

1157

.enc_type = (ENC_TYPE_ALG_DES |

1158

ENC_TYPE_CHAINING_CBC),

1158

ENC_TYPE_CHAINING_CBC),

1159

.ablkcipher = {

1159

.ablkcipher = {

1160

.ivsize = DES_BLOCK_SIZE,

1160

.ivsize = DES_BLOCK_SIZE,

1161

.min_keysize = DES_KEY_SIZE,

1161

.min_keysize = DES_KEY_SIZE,

1162

.max_keysize = DES_KEY_SIZE,

1162

.max_keysize = DES_KEY_SIZE,

1163

.setkey = n2_des_setkey,

1163

.setkey = n2_des_setkey,

1164

.encrypt = n2_encrypt_chaining,

1164

.encrypt = n2_encrypt_chaining,

1165

.decrypt = n2_decrypt_chaining,

1165

.decrypt = n2_decrypt_chaining,

1166

},

1166

},

1167

},

1167

},

1168

{ .name = "cfb(des)",

1168

{ .name = "cfb(des)",

1169

.drv_name = "cfb-des",

1169

.drv_name = "cfb-des",

1170

.block_size = DES_BLOCK_SIZE,

1170

.block_size = DES_BLOCK_SIZE,

1171

.enc_type = (ENC_TYPE_ALG_DES |

1171

.enc_type = (ENC_TYPE_ALG_DES |

1172

ENC_TYPE_CHAINING_CFB),

1172

ENC_TYPE_CHAINING_CFB),

1173

.ablkcipher = {

1173

.ablkcipher = {

1174

.min_keysize = DES_KEY_SIZE,

1174

.min_keysize = DES_KEY_SIZE,

1175

.max_keysize = DES_KEY_SIZE,

1175

.max_keysize = DES_KEY_SIZE,

1176

.setkey = n2_des_setkey,

1176

.setkey = n2_des_setkey,

1177

.encrypt = n2_encrypt_chaining,

1177

.encrypt = n2_encrypt_chaining,

1178

.decrypt = n2_decrypt_chaining,

1178

.decrypt = n2_decrypt_chaining,

1179

},

1179

},

1180

},

1180

},

1181

1182

/* 3DES: ECB CBC and CFB are supported */

1182

/* 3DES: ECB CBC and CFB are supported */

1183

{ .name = "ecb(des3_ede)",

1183

{ .name = "ecb(des3_ede)",

1184

.drv_name = "ecb-3des",

1184

.drv_name = "ecb-3des",

1185

.block_size = DES_BLOCK_SIZE,

1185

.block_size = DES_BLOCK_SIZE,

1186

.enc_type = (ENC_TYPE_ALG_3DES |

1186

.enc_type = (ENC_TYPE_ALG_3DES |

1187

ENC_TYPE_CHAINING_ECB),

1187

ENC_TYPE_CHAINING_ECB),

1188

.ablkcipher = {

1188

.ablkcipher = {

1189

.min_keysize = 3 * DES_KEY_SIZE,

1189

.min_keysize = 3 * DES_KEY_SIZE,

1190

.max_keysize = 3 * DES_KEY_SIZE,

1190

.max_keysize = 3 * DES_KEY_SIZE,

1191

.setkey = n2_3des_setkey,

1191

.setkey = n2_3des_setkey,

1192

.encrypt = n2_encrypt_ecb,

1192

.encrypt = n2_encrypt_ecb,

1193

.decrypt = n2_decrypt_ecb,

1193

.decrypt = n2_decrypt_ecb,

1194

},

1194

},

1195

},

1195

},

1196

{ .name = "cbc(des3_ede)",

1196

{ .name = "cbc(des3_ede)",

1197

.drv_name = "cbc-3des",

1197

.drv_name = "cbc-3des",

1198

.block_size = DES_BLOCK_SIZE,

1198

.block_size = DES_BLOCK_SIZE,

1199

.enc_type = (ENC_TYPE_ALG_3DES |

1199

.enc_type = (ENC_TYPE_ALG_3DES |

1200

ENC_TYPE_CHAINING_CBC),

1200

ENC_TYPE_CHAINING_CBC),

1201

.ablkcipher = {

1201

.ablkcipher = {

1202

.ivsize = DES_BLOCK_SIZE,

1202

.ivsize = DES_BLOCK_SIZE,

1203

.min_keysize = 3 * DES_KEY_SIZE,

1203

.min_keysize = 3 * DES_KEY_SIZE,

1204

.max_keysize = 3 * DES_KEY_SIZE,

1204

.max_keysize = 3 * DES_KEY_SIZE,

1205

.setkey = n2_3des_setkey,

1205

.setkey = n2_3des_setkey,

1206

.encrypt = n2_encrypt_chaining,

1206

.encrypt = n2_encrypt_chaining,

1207

.decrypt = n2_decrypt_chaining,

1207

.decrypt = n2_decrypt_chaining,

1208

},

1208

},

1209

},

1209

},

1210

{ .name = "cfb(des3_ede)",

1210

{ .name = "cfb(des3_ede)",

1211

.drv_name = "cfb-3des",

1211

.drv_name = "cfb-3des",

1212

.block_size = DES_BLOCK_SIZE,

1212

.block_size = DES_BLOCK_SIZE,

1213

.enc_type = (ENC_TYPE_ALG_3DES |

1213

.enc_type = (ENC_TYPE_ALG_3DES |

1214

ENC_TYPE_CHAINING_CFB),

1214

ENC_TYPE_CHAINING_CFB),

1215

.ablkcipher = {

1215

.ablkcipher = {

1216

.min_keysize = 3 * DES_KEY_SIZE,

1216

.min_keysize = 3 * DES_KEY_SIZE,

1217

.max_keysize = 3 * DES_KEY_SIZE,

1217

.max_keysize = 3 * DES_KEY_SIZE,

1218

.setkey = n2_3des_setkey,

1218

.setkey = n2_3des_setkey,

1219

.encrypt = n2_encrypt_chaining,

1219

.encrypt = n2_encrypt_chaining,

1220

.decrypt = n2_decrypt_chaining,

1220

.decrypt = n2_decrypt_chaining,

1221

},

1221

},

1222

},

1222

},

1223

/* AES: ECB CBC and CTR are supported */

1223

/* AES: ECB CBC and CTR are supported */

1224

{ .name = "ecb(aes)",

1224

{ .name = "ecb(aes)",

1225

.drv_name = "ecb-aes",

1225

.drv_name = "ecb-aes",

1226

.block_size = AES_BLOCK_SIZE,

1226

.block_size = AES_BLOCK_SIZE,

1227

.enc_type = (ENC_TYPE_ALG_AES128 |

1227

.enc_type = (ENC_TYPE_ALG_AES128 |

1228

ENC_TYPE_CHAINING_ECB),

1228

ENC_TYPE_CHAINING_ECB),

1229

.ablkcipher = {

1229

.ablkcipher = {

1230

.min_keysize = AES_MIN_KEY_SIZE,

1230

.min_keysize = AES_MIN_KEY_SIZE,

1231

.max_keysize = AES_MAX_KEY_SIZE,

1231

.max_keysize = AES_MAX_KEY_SIZE,

1232

.setkey = n2_aes_setkey,

1232

.setkey = n2_aes_setkey,

1233

.encrypt = n2_encrypt_ecb,

1233

.encrypt = n2_encrypt_ecb,

1234

.decrypt = n2_decrypt_ecb,

1234

.decrypt = n2_decrypt_ecb,

1235

},

1235

},

1236

},

1236

},

1237

{ .name = "cbc(aes)",

1237

{ .name = "cbc(aes)",

1238

.drv_name = "cbc-aes",

1238

.drv_name = "cbc-aes",

1239

.block_size = AES_BLOCK_SIZE,

1239

.block_size = AES_BLOCK_SIZE,

1240

.enc_type = (ENC_TYPE_ALG_AES128 |

1240

.enc_type = (ENC_TYPE_ALG_AES128 |

1241

ENC_TYPE_CHAINING_CBC),

1241

ENC_TYPE_CHAINING_CBC),

1242

.ablkcipher = {

1242

.ablkcipher = {

1243

.ivsize = AES_BLOCK_SIZE,

1243

.ivsize = AES_BLOCK_SIZE,

1244

.min_keysize = AES_MIN_KEY_SIZE,

1244

.min_keysize = AES_MIN_KEY_SIZE,

1245

.max_keysize = AES_MAX_KEY_SIZE,

1245

.max_keysize = AES_MAX_KEY_SIZE,

1246

.setkey = n2_aes_setkey,

1246

.setkey = n2_aes_setkey,

1247

.encrypt = n2_encrypt_chaining,

1247

.encrypt = n2_encrypt_chaining,

1248

.decrypt = n2_decrypt_chaining,

1248

.decrypt = n2_decrypt_chaining,

1249

},

1249

},

1250

},

1250

},

1251

{ .name = "ctr(aes)",

1251

{ .name = "ctr(aes)",

1252

.drv_name = "ctr-aes",

1252

.drv_name = "ctr-aes",

1253

.block_size = AES_BLOCK_SIZE,

1253

.block_size = AES_BLOCK_SIZE,

1254

.enc_type = (ENC_TYPE_ALG_AES128 |

1254

.enc_type = (ENC_TYPE_ALG_AES128 |

1255

ENC_TYPE_CHAINING_COUNTER),

1255

ENC_TYPE_CHAINING_COUNTER),

1256

.ablkcipher = {

1256

.ablkcipher = {

1257

.ivsize = AES_BLOCK_SIZE,

1257

.ivsize = AES_BLOCK_SIZE,

1258

.min_keysize = AES_MIN_KEY_SIZE,

1258

.min_keysize = AES_MIN_KEY_SIZE,

1259

.max_keysize = AES_MAX_KEY_SIZE,

1259

.max_keysize = AES_MAX_KEY_SIZE,

1260

.setkey = n2_aes_setkey,

1260

.setkey = n2_aes_setkey,

1261

.encrypt = n2_encrypt_chaining,

1261

.encrypt = n2_encrypt_chaining,

1262

.decrypt = n2_encrypt_chaining,

1262

.decrypt = n2_encrypt_chaining,

1263

},

1263

},

1264

},

1264

},

1265

1266

};

1266

};

1267

#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)

1267

#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)

1268

1269

static LIST_HEAD(cipher_algs);

1269

static LIST_HEAD(cipher_algs);

1270

1271

struct n2_hash_tmpl {

1271

struct n2_hash_tmpl {

1272

const char *name;

1272

const char *name;

1273

const char *hash_zero;

1273

const char *hash_zero;

1274

const u32 *hash_init;

1274

const u32 *hash_init;

1275

u8 hw_op_hashsz;

1275

u8 hw_op_hashsz;

1276

u8 digest_size;

1276

u8 digest_size;

1277

u8 block_size;

1277

u8 block_size;

1278

u8 auth_type;

1278

u8 auth_type;

1279

u8 hmac_type;

1279

u8 hmac_type;

1280

};

1280

};

1281

1282

static const char md5_zero[MD5_DIGEST_SIZE] = {

1282

static const char md5_zero[MD5_DIGEST_SIZE] = {

1283

0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,

1283

0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,

1284

0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,

1284

0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,

1285

};

1285

};

1286

static const u32 md5_init[MD5_HASH_WORDS] = {

1286

static const u32 md5_init[MD5_HASH_WORDS] = {

1287

cpu_to_le32(0x67452301),

1287

cpu_to_le32(0x67452301),

1288

cpu_to_le32(0xefcdab89),

1288

cpu_to_le32(0xefcdab89),

1289

cpu_to_le32(0x98badcfe),

1289

cpu_to_le32(0x98badcfe),

1290

cpu_to_le32(0x10325476),

1290

cpu_to_le32(0x10325476),

1291

};

1291

};

1292

static const char sha1_zero[SHA1_DIGEST_SIZE] = {

1292

static const char sha1_zero[SHA1_DIGEST_SIZE] = {

1293

0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,

1293

0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,

1294

0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,

1294

0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,

1295

0x07, 0x09

1295

0x07, 0x09

1296

};

1296

};

1297

static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {

1297

static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {

1298

SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,

1298

SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,

1299

};

1299

};

1300

static const char sha256_zero[SHA256_DIGEST_SIZE] = {

1300

static const char sha256_zero[SHA256_DIGEST_SIZE] = {

1301

0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,

1301

0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,

1302

0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,

1302

0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,

1303

0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,

1303

0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,

1304

0x1b, 0x78, 0x52, 0xb8, 0x55

1304

0x1b, 0x78, 0x52, 0xb8, 0x55

1305

};

1305

};

1306

static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {

1306

static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {

1307

SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,

1307

SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,

1308

SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,

1308

SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,

1309

};

1309

};

1310

static const char sha224_zero[SHA224_DIGEST_SIZE] = {

1310

static const char sha224_zero[SHA224_DIGEST_SIZE] = {

1311

0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,

1311

0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,

1312

0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,

1312

0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,

1313

0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,

1313

0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,

1314

0x2f

1314

0x2f

1315

};

1315

};

1316

static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {

1316

static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {

1317

SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,

1317

SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,

1318

SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,

1318

SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,

1319

};

1319

};

1320

1321

static const struct n2_hash_tmpl hash_tmpls[] = {

1321

static const struct n2_hash_tmpl hash_tmpls[] = {

1322

{ .name = "md5",

1322

{ .name = "md5",

1323

.hash_zero = md5_zero,

1323

.hash_zero = md5_zero,

1324

.hash_init = md5_init,

1324

.hash_init = md5_init,

1325

.auth_type = AUTH_TYPE_MD5,

1325

.auth_type = AUTH_TYPE_MD5,

1326

.hmac_type = AUTH_TYPE_HMAC_MD5,

1326

.hmac_type = AUTH_TYPE_HMAC_MD5,

1327

.hw_op_hashsz = MD5_DIGEST_SIZE,

1327

.hw_op_hashsz = MD5_DIGEST_SIZE,

1328

.digest_size = MD5_DIGEST_SIZE,

1328

.digest_size = MD5_DIGEST_SIZE,

1329

.block_size = MD5_HMAC_BLOCK_SIZE },

1329

.block_size = MD5_HMAC_BLOCK_SIZE },

1330

{ .name = "sha1",

1330

{ .name = "sha1",

1331

.hash_zero = sha1_zero,

1331

.hash_zero = sha1_zero,

1332

.hash_init = sha1_init,

1332

.hash_init = sha1_init,

1333

.auth_type = AUTH_TYPE_SHA1,

1333

.auth_type = AUTH_TYPE_SHA1,

1334

.hmac_type = AUTH_TYPE_HMAC_SHA1,

1334

.hmac_type = AUTH_TYPE_HMAC_SHA1,

1335

.hw_op_hashsz = SHA1_DIGEST_SIZE,

1335

.hw_op_hashsz = SHA1_DIGEST_SIZE,

1336

.digest_size = SHA1_DIGEST_SIZE,

1336

.digest_size = SHA1_DIGEST_SIZE,

1337

.block_size = SHA1_BLOCK_SIZE },

1337

.block_size = SHA1_BLOCK_SIZE },

1338

{ .name = "sha256",

1338

{ .name = "sha256",

1339

.hash_zero = sha256_zero,

1339

.hash_zero = sha256_zero,

1340

.hash_init = sha256_init,

1340

.hash_init = sha256_init,

1341

.auth_type = AUTH_TYPE_SHA256,

1341

.auth_type = AUTH_TYPE_SHA256,

1342

.hmac_type = AUTH_TYPE_HMAC_SHA256,

1342

.hmac_type = AUTH_TYPE_HMAC_SHA256,

1343

.hw_op_hashsz = SHA256_DIGEST_SIZE,

1343

.hw_op_hashsz = SHA256_DIGEST_SIZE,

1344

.digest_size = SHA256_DIGEST_SIZE,

1344

.digest_size = SHA256_DIGEST_SIZE,

1345

.block_size = SHA256_BLOCK_SIZE },

1345

.block_size = SHA256_BLOCK_SIZE },

1346

{ .name = "sha224",

1346

{ .name = "sha224",

1347

.hash_zero = sha224_zero,

1347

.hash_zero = sha224_zero,

1348

.hash_init = sha224_init,

1348

.hash_init = sha224_init,

1349

.auth_type = AUTH_TYPE_SHA256,

1349

.auth_type = AUTH_TYPE_SHA256,

1350

.hmac_type = AUTH_TYPE_RESERVED,

1350

.hmac_type = AUTH_TYPE_RESERVED,

1351

.hw_op_hashsz = SHA256_DIGEST_SIZE,

1351

.hw_op_hashsz = SHA256_DIGEST_SIZE,

1352

.digest_size = SHA224_DIGEST_SIZE,

1352

.digest_size = SHA224_DIGEST_SIZE,

1353

.block_size = SHA224_BLOCK_SIZE },

1353

.block_size = SHA224_BLOCK_SIZE },

1354

};

1354

};

1355

#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)

1355

#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)

1356

1357

static LIST_HEAD(ahash_algs);

1357

static LIST_HEAD(ahash_algs);

1358

static LIST_HEAD(hmac_algs);

1358

static LIST_HEAD(hmac_algs);

1359

1360

static int algs_registered;

1360

static int algs_registered;

1361

1362

static void __n2_unregister_algs(void)

1362

static void __n2_unregister_algs(void)

1363

{

1363

{

1364

struct n2_cipher_alg *cipher, *cipher_tmp;

1364

struct n2_cipher_alg *cipher, *cipher_tmp;

1365

struct n2_ahash_alg *alg, *alg_tmp;

1365

struct n2_ahash_alg *alg, *alg_tmp;

1366

struct n2_hmac_alg *hmac, *hmac_tmp;

1366

struct n2_hmac_alg *hmac, *hmac_tmp;

1367

1368

list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {

1368

list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {

1369

crypto_unregister_alg(&cipher->alg);

1369

crypto_unregister_alg(&cipher->alg);

1370

list_del(&cipher->entry);

1370

list_del(&cipher->entry);

1371

kfree(cipher);

1371

kfree(cipher);

1372

}

1372

}

1373

list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {

1373

list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {

1374

crypto_unregister_ahash(&hmac->derived.alg);

1374

crypto_unregister_ahash(&hmac->derived.alg);

1375

list_del(&hmac->derived.entry);

1375

list_del(&hmac->derived.entry);

1376

kfree(hmac);

1376

kfree(hmac);

1377

}

1377

}

1378

list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {

1378

list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {

1379

crypto_unregister_ahash(&alg->alg);

1379

crypto_unregister_ahash(&alg->alg);

1380

list_del(&alg->entry);

1380

list_del(&alg->entry);

1381

kfree(alg);

1381

kfree(alg);

1382

}

1382

}

1383

}

1383

}

1384

1385

static int n2_cipher_cra_init(struct crypto_tfm *tfm)

1385

static int n2_cipher_cra_init(struct crypto_tfm *tfm)

1386

{

1386

{

1387

tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);

1387

tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);

1388

return 0;

1388

return 0;

1389

}

1389

}

1390

1391

static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)

1391

static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)

1392

{

1392

{

1393

struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1393

struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1394

struct crypto_alg *alg;

1394

struct crypto_alg *alg;

1395

int err;

1395

int err;

1396

1397

if (!p)

1397

if (!p)

1398

return -ENOMEM;

1398

return -ENOMEM;

1399

1400

alg = &p->alg;

1400

alg = &p->alg;

1401

1402

snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);

1402

snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);

1403

snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);

1403

snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);

1404

alg->cra_priority = N2_CRA_PRIORITY;

1404

alg->cra_priority = N2_CRA_PRIORITY;

1405

alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;

1405

alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;

1406

alg->cra_blocksize = tmpl->block_size;

1406

alg->cra_blocksize = tmpl->block_size;

1407

p->enc_type = tmpl->enc_type;

1407

p->enc_type = tmpl->enc_type;

1408

alg->cra_ctxsize = sizeof(struct n2_cipher_context);

1408

alg->cra_ctxsize = sizeof(struct n2_cipher_context);

1409

alg->cra_type = &crypto_ablkcipher_type;

1409

alg->cra_type = &crypto_ablkcipher_type;

1410

alg->cra_u.ablkcipher = tmpl->ablkcipher;

1410

alg->cra_u.ablkcipher = tmpl->ablkcipher;

1411

alg->cra_init = n2_cipher_cra_init;

1411

alg->cra_init = n2_cipher_cra_init;

1412

alg->cra_module = THIS_MODULE;

1412

alg->cra_module = THIS_MODULE;

1413

1414

list_add(&p->entry, &cipher_algs);

1414

list_add(&p->entry, &cipher_algs);

1415

err = crypto_register_alg(alg);

1415

err = crypto_register_alg(alg);

1416

if (err) {

1416

if (err) {

1417

pr_err("%s alg registration failed\n", alg->cra_name);

1417

pr_err("%s alg registration failed\n", alg->cra_name);

1418

list_del(&p->entry);

1418

list_del(&p->entry);

1419

kfree(p);

1419

kfree(p);

1420

} else {

1420

} else {

1421

pr_info("%s alg registered\n", alg->cra_name);

1421

pr_info("%s alg registered\n", alg->cra_name);

1422

}

1422

}

1423

return err;

1423

return err;

1424

}

1424

}

1425

1426

static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)

1426

static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)

1427

{

1427

{

1428

struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1428

struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1429

struct ahash_alg *ahash;

1429

struct ahash_alg *ahash;

1430

struct crypto_alg *base;

1430

struct crypto_alg *base;

1431

int err;

1431

int err;

1432

1433

if (!p)

1433

if (!p)

1434

return -ENOMEM;

1434

return -ENOMEM;

1435

1436

p->child_alg = n2ahash->alg.halg.base.cra_name;

1436

p->child_alg = n2ahash->alg.halg.base.cra_name;

1437

memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));

1437

memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));

1438

INIT_LIST_HEAD(&p->derived.entry);

1438

INIT_LIST_HEAD(&p->derived.entry);

1439

1440

ahash = &p->derived.alg;

1440

ahash = &p->derived.alg;

1441

ahash->digest = n2_hmac_async_digest;

1441

ahash->digest = n2_hmac_async_digest;

1442

ahash->setkey = n2_hmac_async_setkey;

1442

ahash->setkey = n2_hmac_async_setkey;

1443

1444

base = &ahash->halg.base;

1444

base = &ahash->halg.base;

1445

snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);

1445

snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);

1446

snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);

1446

snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);

1447

1448

base->cra_ctxsize = sizeof(struct n2_hmac_ctx);

1448

base->cra_ctxsize = sizeof(struct n2_hmac_ctx);

1449

base->cra_init = n2_hmac_cra_init;

1449

base->cra_init = n2_hmac_cra_init;

1450

base->cra_exit = n2_hmac_cra_exit;

1450

base->cra_exit = n2_hmac_cra_exit;

1451

1452

list_add(&p->derived.entry, &hmac_algs);

1452

list_add(&p->derived.entry, &hmac_algs);

1453

err = crypto_register_ahash(ahash);

1453

err = crypto_register_ahash(ahash);

1454

if (err) {

1454

if (err) {

1455

pr_err("%s alg registration failed\n", base->cra_name);

1455

pr_err("%s alg registration failed\n", base->cra_name);

1456

list_del(&p->derived.entry);

1456

list_del(&p->derived.entry);

1457

kfree(p);

1457

kfree(p);

1458

} else {

1458

} else {

1459

pr_info("%s alg registered\n", base->cra_name);

1459

pr_info("%s alg registered\n", base->cra_name);

1460

}

1460

}

1461

return err;

1461

return err;

1462

}

1462

}

1463

1464

static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)

1464

static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)

1465

{

1465

{

1466

struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1466

struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);

1467

struct hash_alg_common *halg;

1467

struct hash_alg_common *halg;

1468

struct crypto_alg *base;

1468

struct crypto_alg *base;

1469

struct ahash_alg *ahash;

1469

struct ahash_alg *ahash;

1470

int err;

1470

int err;

1471

1472

if (!p)

1472

if (!p)

1473

return -ENOMEM;

1473

return -ENOMEM;

1474

1475

p->hash_zero = tmpl->hash_zero;

1475

p->hash_zero = tmpl->hash_zero;

1476

p->hash_init = tmpl->hash_init;

1476

p->hash_init = tmpl->hash_init;

1477

p->auth_type = tmpl->auth_type;

1477

p->auth_type = tmpl->auth_type;

1478

p->hmac_type = tmpl->hmac_type;

1478

p->hmac_type = tmpl->hmac_type;

1479

p->hw_op_hashsz = tmpl->hw_op_hashsz;

1479

p->hw_op_hashsz = tmpl->hw_op_hashsz;

1480

p->digest_size = tmpl->digest_size;

1480

p->digest_size = tmpl->digest_size;

1481

1482

ahash = &p->alg;

1482

ahash = &p->alg;

1483

ahash->init = n2_hash_async_init;

1483

ahash->init = n2_hash_async_init;

1484

ahash->update = n2_hash_async_update;

1484

ahash->update = n2_hash_async_update;

1485

ahash->final = n2_hash_async_final;

1485

ahash->final = n2_hash_async_final;

1486

ahash->finup = n2_hash_async_finup;

1486

ahash->finup = n2_hash_async_finup;

1487

ahash->digest = n2_hash_async_digest;

1487

ahash->digest = n2_hash_async_digest;

1488

1489

halg = &ahash->halg;

1489

halg = &ahash->halg;

1490

halg->digestsize = tmpl->digest_size;

1490

halg->digestsize = tmpl->digest_size;

1491

1492

base = &halg->base;

1492

base = &halg->base;

1493

snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);

1493

snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);

1494

snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);

1494

snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);

1495

base->cra_priority = N2_CRA_PRIORITY;

1495

base->cra_priority = N2_CRA_PRIORITY;

1496

base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;

1496

base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;

1497

base->cra_blocksize = tmpl->block_size;

1497

base->cra_blocksize = tmpl->block_size;

1498

base->cra_ctxsize = sizeof(struct n2_hash_ctx);

1498

base->cra_ctxsize = sizeof(struct n2_hash_ctx);

1499

base->cra_module = THIS_MODULE;

1499

base->cra_module = THIS_MODULE;

1500

base->cra_init = n2_hash_cra_init;

1500

base->cra_init = n2_hash_cra_init;

1501

base->cra_exit = n2_hash_cra_exit;

1501

base->cra_exit = n2_hash_cra_exit;

1502

1503

list_add(&p->entry, &ahash_algs);

1503

list_add(&p->entry, &ahash_algs);

1504

err = crypto_register_ahash(ahash);

1504

err = crypto_register_ahash(ahash);

1505

if (err) {

1505

if (err) {

1506

pr_err("%s alg registration failed\n", base->cra_name);

1506

pr_err("%s alg registration failed\n", base->cra_name);

1507

list_del(&p->entry);

1507

list_del(&p->entry);

1508

kfree(p);

1508

kfree(p);

1509

} else {

1509

} else {

1510

pr_info("%s alg registered\n", base->cra_name);

1510

pr_info("%s alg registered\n", base->cra_name);

1511

}

1511

}

1512

if (!err && p->hmac_type != AUTH_TYPE_RESERVED)

1512

if (!err && p->hmac_type != AUTH_TYPE_RESERVED)

1513

err = __n2_register_one_hmac(p);

1513

err = __n2_register_one_hmac(p);

1514

return err;

1514

return err;

1515

}

1515

}

1516

1517

static int __devinit n2_register_algs(void)

1517

static int __devinit n2_register_algs(void)

1518

{

1518

{

1519

int i, err = 0;

1519

int i, err = 0;

1520

1521

mutex_lock(&spu_lock);

1521

mutex_lock(&spu_lock);

1522

if (algs_registered++)

1522

if (algs_registered++)

1523

goto out;

1523

goto out;

1524

1525

for (i = 0; i < NUM_HASH_TMPLS; i++) {

1525

for (i = 0; i < NUM_HASH_TMPLS; i++) {

1526

err = __n2_register_one_ahash(&hash_tmpls[i]);

1526

err = __n2_register_one_ahash(&hash_tmpls[i]);

1527

if (err) {

1527

if (err) {

1528

__n2_unregister_algs();

1528

__n2_unregister_algs();

1529

goto out;

1529

goto out;

1530

}

1530

}

1531

}

1531

}

1532

for (i = 0; i < NUM_CIPHER_TMPLS; i++) {

1532

for (i = 0; i < NUM_CIPHER_TMPLS; i++) {

1533

err = __n2_register_one_cipher(&cipher_tmpls[i]);

1533

err = __n2_register_one_cipher(&cipher_tmpls[i]);

1534

if (err) {

1534

if (err) {

1535

__n2_unregister_algs();

1535

__n2_unregister_algs();

1536

goto out;

1536

goto out;

1537

}

1537

}

1538

}

1538

}

1539

1540

out:

1540

out:

1541

mutex_unlock(&spu_lock);

1541

mutex_unlock(&spu_lock);

1542

return err;

1542

return err;

1543

}

1543

}

1544

1545

static void __devexit n2_unregister_algs(void)

1545

static void __devexit n2_unregister_algs(void)

1546

{

1546

{

1547

mutex_lock(&spu_lock);

1547

mutex_lock(&spu_lock);

1548

if (!--algs_registered)

1548

if (!--algs_registered)

1549

__n2_unregister_algs();

1549

__n2_unregister_algs();

1550

mutex_unlock(&spu_lock);

1550

mutex_unlock(&spu_lock);

1551

}

1551

}

1552

1553

/* To map CWQ queues to interrupt sources, the hypervisor API provides

1553

/* To map CWQ queues to interrupt sources, the hypervisor API provides

1554

* a devino. This isn't very useful to us because all of the

1554

* a devino. This isn't very useful to us because all of the

1555

* interrupts listed in the device_node have been translated to

1555

* interrupts listed in the device_node have been translated to

1556

* Linux virtual IRQ cookie numbers.

1556

* Linux virtual IRQ cookie numbers.

1557

*

1557

*

1558

* So we have to back-translate, going through the 'intr' and 'ino'

1558

* So we have to back-translate, going through the 'intr' and 'ino'

1559

* property tables of the n2cp MDESC node, matching it with the OF

1559

* property tables of the n2cp MDESC node, matching it with the OF

1560

* 'interrupts' property entries, in order to to figure out which

1560

* 'interrupts' property entries, in order to to figure out which

1561

* devino goes to which already-translated IRQ.

1561

* devino goes to which already-translated IRQ.

1562

*/

1562

*/

1563

static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,

1563

static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,

1564

unsigned long dev_ino)

1564

unsigned long dev_ino)

1565

{

1565

{

1566

const unsigned int *dev_intrs;

1566

const unsigned int *dev_intrs;

1567

unsigned int intr;

1567

unsigned int intr;

1568

int i;

1568

int i;

1569

1570

for (i = 0; i < ip->num_intrs; i++) {

1570

for (i = 0; i < ip->num_intrs; i++) {

1571

if (ip->ino_table[i].ino == dev_ino)

1571

if (ip->ino_table[i].ino == dev_ino)

1572

break;

1572

break;

1573

}

1573

}

1574

if (i == ip->num_intrs)

1574

if (i == ip->num_intrs)

1575

return -ENODEV;

1575

return -ENODEV;

1576

1577

intr = ip->ino_table[i].intr;

1577

intr = ip->ino_table[i].intr;

1578

1579

dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);

1579

dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);

1580

if (!dev_intrs)

1580

if (!dev_intrs)

1581

return -ENODEV;

1581

return -ENODEV;

1582

1583

for (i = 0; i < dev->archdata.num_irqs; i++) {

1583

for (i = 0; i < dev->archdata.num_irqs; i++) {

1584

if (dev_intrs[i] == intr)

1584

if (dev_intrs[i] == intr)

1585

return i;

1585

return i;

1586

}

1586

}

1587

1588

return -ENODEV;

1588

return -ENODEV;

1589

}

1589

}

1590

1591

static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,

1591

static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,

1592

const char *irq_name, struct spu_queue *p,

1592

const char *irq_name, struct spu_queue *p,

1593

irq_handler_t handler)

1593

irq_handler_t handler)

1594

{

1594

{

1595

unsigned long herr;

1595

unsigned long herr;

1596

int index;

1596

int index;

1597

1598

herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);

1598

herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);

1599

if (herr)

1599

if (herr)

1600

return -EINVAL;

1600

return -EINVAL;

1601

1602

index = find_devino_index(dev, ip, p->devino);

1602

index = find_devino_index(dev, ip, p->devino);

1603

if (index < 0)

1603

if (index < 0)

1604

return index;

1604

return index;

1605

1606

p->irq = dev->archdata.irqs[index];

1606

p->irq = dev->archdata.irqs[index];

1607

1608

sprintf(p->irq_name, "%s-%d", irq_name, index);

1608

sprintf(p->irq_name, "%s-%d", irq_name, index);

1609

1610

return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,

1610

return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,

1611

p->irq_name, p);

1611

p->irq_name, p);

1612

}

1612

}

1613

1614

static struct kmem_cache *queue_cache[2];

1614

static struct kmem_cache *queue_cache[2];

1615

1616

static void *new_queue(unsigned long q_type)

1616

static void *new_queue(unsigned long q_type)

1617

{

1617

{

1618

return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);

1618

return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);

1619

}

1619

}

1620

1621

static void free_queue(void *p, unsigned long q_type)

1621

static void free_queue(void *p, unsigned long q_type)

1622

{

1622

{

1623

return kmem_cache_free(queue_cache[q_type - 1], p);

1623

return kmem_cache_free(queue_cache[q_type - 1], p);

1624

}

1624

}

1625

1626

static int queue_cache_init(void)

1626

static int queue_cache_init(void)

1627

{

1627

{

1628

if (!queue_cache[HV_NCS_QTYPE_MAU - 1])

1628

if (!queue_cache[HV_NCS_QTYPE_MAU - 1])

1629

queue_cache[HV_NCS_QTYPE_MAU - 1] =

1629

queue_cache[HV_NCS_QTYPE_MAU - 1] =

1630

kmem_cache_create("mau_queue",

1630

kmem_cache_create("mau_queue",

1631

(MAU_NUM_ENTRIES *

1631

(MAU_NUM_ENTRIES *

1632

MAU_ENTRY_SIZE),

1632

MAU_ENTRY_SIZE),

1633

MAU_ENTRY_SIZE, 0, NULL);

1633

MAU_ENTRY_SIZE, 0, NULL);

1634

if (!queue_cache[HV_NCS_QTYPE_MAU - 1])

1634

if (!queue_cache[HV_NCS_QTYPE_MAU - 1])

1635

return -ENOMEM;

1635

return -ENOMEM;

1636

1637

if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])

1637

if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])

1638

queue_cache[HV_NCS_QTYPE_CWQ - 1] =

1638

queue_cache[HV_NCS_QTYPE_CWQ - 1] =

1639

kmem_cache_create("cwq_queue",

1639

kmem_cache_create("cwq_queue",

1640

(CWQ_NUM_ENTRIES *

1640

(CWQ_NUM_ENTRIES *

1641

CWQ_ENTRY_SIZE),

1641

CWQ_ENTRY_SIZE),

1642

CWQ_ENTRY_SIZE, 0, NULL);

1642

CWQ_ENTRY_SIZE, 0, NULL);

1643

if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {

1643

if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {

1644

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);

1644

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);

1645

return -ENOMEM;

1645

return -ENOMEM;

1646

}

1646

}

1647

return 0;

1647

return 0;

1648

}

1648

}

1649

1650

static void queue_cache_destroy(void)

1650

static void queue_cache_destroy(void)

1651

{

1651

{

1652

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);

1652

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);

1653

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);

1653

kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);

1654

}

1654

}

1655

1656

static int spu_queue_register(struct spu_queue *p, unsigned long q_type)

1656

static int spu_queue_register(struct spu_queue *p, unsigned long q_type)

1657

{

1657

{

1658

cpumask_var_t old_allowed;

1658

cpumask_var_t old_allowed;

1659

unsigned long hv_ret;

1659

unsigned long hv_ret;

1660

1661

if (cpumask_empty(&p->sharing))

1661

if (cpumask_empty(&p->sharing))

1662

return -EINVAL;

1662

return -EINVAL;

1663

1664

if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))

1664

if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))

1665

return -ENOMEM;

1665

return -ENOMEM;

1666

1667

cpumask_copy(old_allowed, &current->cpus_allowed);

1667

cpumask_copy(old_allowed, &current->cpus_allowed);

1668

1669

set_cpus_allowed_ptr(current, &p->sharing);

1669

set_cpus_allowed_ptr(current, &p->sharing);

1670

1671

hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),

1671

hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),

1672

CWQ_NUM_ENTRIES, &p->qhandle);

1672

CWQ_NUM_ENTRIES, &p->qhandle);

1673

if (!hv_ret)

1673

if (!hv_ret)

1674

sun4v_ncs_sethead_marker(p->qhandle, 0);

1674

sun4v_ncs_sethead_marker(p->qhandle, 0);

1675

1676

set_cpus_allowed_ptr(current, old_allowed);

1676

set_cpus_allowed_ptr(current, old_allowed);

1677

1678

free_cpumask_var(old_allowed);

1678

free_cpumask_var(old_allowed);

1679

1680

return (hv_ret ? -EINVAL : 0);

1680

return (hv_ret ? -EINVAL : 0);

1681

}

1681

}

1682

1683

static int spu_queue_setup(struct spu_queue *p)

1683

static int spu_queue_setup(struct spu_queue *p)

1684

{

1684

{

1685

int err;

1685

int err;

1686

1687

p->q = new_queue(p->q_type);

1687

p->q = new_queue(p->q_type);

1688

if (!p->q)

1688

if (!p->q)

1689

return -ENOMEM;

1689

return -ENOMEM;

1690

1691

err = spu_queue_register(p, p->q_type);

1691

err = spu_queue_register(p, p->q_type);

1692

if (err) {

1692

if (err) {

1693

free_queue(p->q, p->q_type);

1693

free_queue(p->q, p->q_type);

1694

p->q = NULL;

1694

p->q = NULL;

1695

}

1695

}

1696

1697

return err;

1697

return err;

1698

}

1698

}

1699

1700

static void spu_queue_destroy(struct spu_queue *p)

1700

static void spu_queue_destroy(struct spu_queue *p)

1701

{

1701

{

1702

unsigned long hv_ret;

1702

unsigned long hv_ret;

1703

1704

if (!p->q)

1704

if (!p->q)

1705

return;

1705

return;

1706

1707

hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);

1707

hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);

1708

1709

if (!hv_ret)

1709

if (!hv_ret)

1710

free_queue(p->q, p->q_type);

1710

free_queue(p->q, p->q_type);

1711

}

1711

}

1712

1713

static void spu_list_destroy(struct list_head *list)

1713

static void spu_list_destroy(struct list_head *list)

1714

{

1714

{

1715

struct spu_queue *p, *n;

1715

struct spu_queue *p, *n;

1716

1717

list_for_each_entry_safe(p, n, list, list) {

1717

list_for_each_entry_safe(p, n, list, list) {

1718

int i;

1718

int i;

1719

1720

for (i = 0; i < NR_CPUS; i++) {

1720

for (i = 0; i < NR_CPUS; i++) {

1721

if (cpu_to_cwq[i] == p)

1721

if (cpu_to_cwq[i] == p)

1722

cpu_to_cwq[i] = NULL;

1722

cpu_to_cwq[i] = NULL;

1723

}

1723

}

1724

1725

if (p->irq) {

1725

if (p->irq) {

1726

free_irq(p->irq, p);

1726

free_irq(p->irq, p);

1727

p->irq = 0;

1727

p->irq = 0;

1728

}

1728

}

1729

spu_queue_destroy(p);

1729

spu_queue_destroy(p);

1730

list_del(&p->list);

1730

list_del(&p->list);

1731

kfree(p);

1731

kfree(p);

1732

}

1732

}

1733

}

1733

}

1734

1735

/* Walk the backward arcs of a CWQ 'exec-unit' node,

1735

/* Walk the backward arcs of a CWQ 'exec-unit' node,

1736

* gathering cpu membership information.

1736

* gathering cpu membership information.

1737

*/

1737

*/

1738

static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,

1738

static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,

1739

struct platform_device *dev,

1739

struct platform_device *dev,

1740

u64 node, struct spu_queue *p,

1740

u64 node, struct spu_queue *p,

1741

struct spu_queue **table)

1741

struct spu_queue **table)

1742

{

1742

{

1743

u64 arc;

1743

u64 arc;

1744

1745

mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {

1745

mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {

1746

u64 tgt = mdesc_arc_target(mdesc, arc);

1746

u64 tgt = mdesc_arc_target(mdesc, arc);

1747

const char *name = mdesc_node_name(mdesc, tgt);

1747

const char *name = mdesc_node_name(mdesc, tgt);

1748

const u64 *id;

1748

const u64 *id;

1749

1750

if (strcmp(name, "cpu"))

1750

if (strcmp(name, "cpu"))

1751

continue;

1751

continue;

1752

id = mdesc_get_property(mdesc, tgt, "id", NULL);

1752

id = mdesc_get_property(mdesc, tgt, "id", NULL);

1753

if (table[*id] != NULL) {

1753

if (table[*id] != NULL) {

1754

dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",

1754

dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",

1755

dev->dev.of_node->full_name);

1755

dev->dev.of_node->full_name);

1756

return -EINVAL;

1756

return -EINVAL;

1757

}

1757

}

1758

cpu_set(*id, p->sharing);

1758

cpu_set(*id, p->sharing);

1759

table[*id] = p;

1759

table[*id] = p;

1760

}

1760

}

1761

return 0;

1761

return 0;

1762

}

1762

}

1763

1764

/* Process an 'exec-unit' MDESC node of type 'cwq'. */

1764

/* Process an 'exec-unit' MDESC node of type 'cwq'. */

1765

static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,

1765

static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,

1766

struct platform_device *dev, struct mdesc_handle *mdesc,

1766

struct platform_device *dev, struct mdesc_handle *mdesc,

1767

u64 node, const char *iname, unsigned long q_type,

1767

u64 node, const char *iname, unsigned long q_type,

1768

irq_handler_t handler, struct spu_queue **table)

1768

irq_handler_t handler, struct spu_queue **table)

1769

{

1769

{

1770

struct spu_queue *p;

1770

struct spu_queue *p;

1771

int err;

1771

int err;

1772

1773

p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);

1773

p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);

1774

if (!p) {

1774

if (!p) {

1775

dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",

1775

dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",

1776

dev->dev.of_node->full_name);

1776

dev->dev.of_node->full_name);

1777

return -ENOMEM;

1777

return -ENOMEM;

1778

}

1778

}

1779

1780

cpus_clear(p->sharing);

1780

cpus_clear(p->sharing);

1781

spin_lock_init(&p->lock);

1781

spin_lock_init(&p->lock);

1782

p->q_type = q_type;

1782

p->q_type = q_type;

1783

INIT_LIST_HEAD(&p->jobs);

1783

INIT_LIST_HEAD(&p->jobs);

1784

list_add(&p->list, list);

1784

list_add(&p->list, list);

1785

1786

err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);

1786

err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);

1787

if (err)

1787

if (err)

1788

return err;

1788

return err;

1789

1790

err = spu_queue_setup(p);

1790

err = spu_queue_setup(p);

1791

if (err)

1791

if (err)

1792

return err;

1792

return err;

1793

1794

return spu_map_ino(dev, ip, iname, p, handler);

1794

return spu_map_ino(dev, ip, iname, p, handler);

1795

}

1795

}

1796

1797

static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,

1797

static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,

1798

struct spu_mdesc_info *ip, struct list_head *list,

1798

struct spu_mdesc_info *ip, struct list_head *list,

1799

const char *exec_name, unsigned long q_type,

1799

const char *exec_name, unsigned long q_type,

1800

irq_handler_t handler, struct spu_queue **table)

1800

irq_handler_t handler, struct spu_queue **table)

1801

{

1801

{

1802

int err = 0;

1802

int err = 0;

1803

u64 node;

1803

u64 node;

1804

1805

mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {

1805

mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {

1806

const char *type;

1806

const char *type;

1807

1808

type = mdesc_get_property(mdesc, node, "type", NULL);

1808

type = mdesc_get_property(mdesc, node, "type", NULL);

1809

if (!type || strcmp(type, exec_name))

1809

if (!type || strcmp(type, exec_name))

1810

continue;

1810

continue;

1811

1812

err = handle_exec_unit(ip, list, dev, mdesc, node,

1812

err = handle_exec_unit(ip, list, dev, mdesc, node,

1813

exec_name, q_type, handler, table);

1813

exec_name, q_type, handler, table);

1814

if (err) {

1814

if (err) {

1815

spu_list_destroy(list);

1815

spu_list_destroy(list);

1816

break;

1816

break;

1817

}

1817

}

1818

}

1818

}

1819

1820

return err;

1820

return err;

1821

}

1821

}

1822

1823

static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,

1823

static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,

1824

struct spu_mdesc_info *ip)

1824

struct spu_mdesc_info *ip)

1825

{

1825

{

1826

const u64 *ino;

1826

const u64 *ino;

1827

int ino_len;

1827

int ino_len;

1828

int i;

1828

int i;

1829

1830

ino = mdesc_get_property(mdesc, node, "ino", &ino_len);

1830

ino = mdesc_get_property(mdesc, node, "ino", &ino_len);

1831

if (!ino) {

1831

if (!ino) {

1832

printk("NO 'ino'\n");

1832

printk("NO 'ino'\n");

1833

return -ENODEV;

1833

return -ENODEV;

1834

}

1834

}

1835

1836

ip->num_intrs = ino_len / sizeof(u64);

1836

ip->num_intrs = ino_len / sizeof(u64);

1837

ip->ino_table = kzalloc((sizeof(struct ino_blob) *

1837

ip->ino_table = kzalloc((sizeof(struct ino_blob) *

1838

ip->num_intrs),

1838

ip->num_intrs),

1839

GFP_KERNEL);

1839

GFP_KERNEL);

1840

if (!ip->ino_table)

1840

if (!ip->ino_table)

1841

return -ENOMEM;

1841

return -ENOMEM;

1842

1843

for (i = 0; i < ip->num_intrs; i++) {

1843

for (i = 0; i < ip->num_intrs; i++) {

1844

struct ino_blob *b = &ip->ino_table[i];

1844

struct ino_blob *b = &ip->ino_table[i];

1845

b->intr = i + 1;

1845

b->intr = i + 1;

1846

b->ino = ino[i];

1846

b->ino = ino[i];

1847

}

1847

}

1848

1849

return 0;

1849

return 0;

1850

}

1850

}

1851

1852

static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,

1852

static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,

1853

struct platform_device *dev,

1853

struct platform_device *dev,

1854

struct spu_mdesc_info *ip,

1854

struct spu_mdesc_info *ip,

1855

const char *node_name)

1855

const char *node_name)

1856

{

1856

{

1857

const unsigned int *reg;

1857

const unsigned int *reg;

1858

u64 node;

1858

u64 node;

1859

1860

reg = of_get_property(dev->dev.of_node, "reg", NULL);

1860

reg = of_get_property(dev->dev.of_node, "reg", NULL);

1861

if (!reg)

1861

if (!reg)

1862

return -ENODEV;

1862

return -ENODEV;

1863

1864

mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {

1864

mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {

1865

const char *name;

1865

const char *name;

1866

const u64 *chdl;

1866

const u64 *chdl;

1867

1868

name = mdesc_get_property(mdesc, node, "name", NULL);

1868

name = mdesc_get_property(mdesc, node, "name", NULL);

1869

if (!name || strcmp(name, node_name))

1869

if (!name || strcmp(name, node_name))

1870

continue;

1870

continue;

1871

chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);

1871

chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);

1872

if (!chdl || (*chdl != *reg))

1872

if (!chdl || (*chdl != *reg))

1873

continue;

1873

continue;

1874

ip->cfg_handle = *chdl;

1874

ip->cfg_handle = *chdl;

1875

return get_irq_props(mdesc, node, ip);

1875

return get_irq_props(mdesc, node, ip);

1876

}

1876

}

1877

1878

return -ENODEV;

1878

return -ENODEV;

1879

}

1879

}

1880

1881

static unsigned long n2_spu_hvapi_major;

1881

static unsigned long n2_spu_hvapi_major;

1882

static unsigned long n2_spu_hvapi_minor;

1882

static unsigned long n2_spu_hvapi_minor;

1883

1884

static int __devinit n2_spu_hvapi_register(void)

1884

static int __devinit n2_spu_hvapi_register(void)

1885

{

1885

{

1886

int err;

1886

int err;

1887

1888

n2_spu_hvapi_major = 2;

1888

n2_spu_hvapi_major = 2;

1889

n2_spu_hvapi_minor = 0;

1889

n2_spu_hvapi_minor = 0;

1890

1891

err = sun4v_hvapi_register(HV_GRP_NCS,

1891

err = sun4v_hvapi_register(HV_GRP_NCS,

1892

n2_spu_hvapi_major,

1892

n2_spu_hvapi_major,

1893

&n2_spu_hvapi_minor);

1893

&n2_spu_hvapi_minor);

1894

1895

if (!err)

1895

if (!err)

1896

pr_info("Registered NCS HVAPI version %lu.%lu\n",

1896

pr_info("Registered NCS HVAPI version %lu.%lu\n",

1897

n2_spu_hvapi_major,

1897

n2_spu_hvapi_major,

1898

n2_spu_hvapi_minor);

1898

n2_spu_hvapi_minor);

1899

1900

return err;

1900

return err;

1901

}

1901

}

1902

1903

static void n2_spu_hvapi_unregister(void)

1903

static void n2_spu_hvapi_unregister(void)

1904

{

1904

{

1905

sun4v_hvapi_unregister(HV_GRP_NCS);

1905

sun4v_hvapi_unregister(HV_GRP_NCS);

1906

}

1906

}

1907

1908

static int global_ref;

1908

static int global_ref;

1909

1910

static int __devinit grab_global_resources(void)

1910

static int __devinit grab_global_resources(void)

1911

{

1911

{

1912

int err = 0;

1912

int err = 0;

1913

1914

mutex_lock(&spu_lock);

1914

mutex_lock(&spu_lock);

1915

1916

if (global_ref++)

1916

if (global_ref++)

1917

goto out;

1917

goto out;

1918

1919

err = n2_spu_hvapi_register();

1919

err = n2_spu_hvapi_register();

1920

if (err)

1920

if (err)

1921

goto out;

1921

goto out;

1922

1923

err = queue_cache_init();

1923

err = queue_cache_init();

1924

if (err)

1924

if (err)

1925

goto out_hvapi_release;

1925

goto out_hvapi_release;

1926

1927

err = -ENOMEM;

1927

err = -ENOMEM;

1928

cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,

1928

cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,

1929

GFP_KERNEL);

1929

GFP_KERNEL);

1930

if (!cpu_to_cwq)

1930

if (!cpu_to_cwq)

1931

goto out_queue_cache_destroy;

1931

goto out_queue_cache_destroy;

1932

1933

cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,

1933

cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,

1934

GFP_KERNEL);

1934

GFP_KERNEL);

1935

if (!cpu_to_mau)

1935

if (!cpu_to_mau)

1936

goto out_free_cwq_table;

1936

goto out_free_cwq_table;

1937

1938

err = 0;

1938

err = 0;

1939

1940

out:

1940

out:

1941

if (err)

1941

if (err)

1942

global_ref--;

1942

global_ref--;

1943

mutex_unlock(&spu_lock);

1943

mutex_unlock(&spu_lock);

1944

return err;

1944

return err;

1945

1946

out_free_cwq_table:

1946

out_free_cwq_table:

1947

kfree(cpu_to_cwq);

1947

kfree(cpu_to_cwq);

1948

cpu_to_cwq = NULL;

1948

cpu_to_cwq = NULL;

1949

1950

out_queue_cache_destroy:

1950

out_queue_cache_destroy:

1951

queue_cache_destroy();

1951

queue_cache_destroy();

1952

1953

out_hvapi_release:

1953

out_hvapi_release:

1954

n2_spu_hvapi_unregister();

1954

n2_spu_hvapi_unregister();

1955

goto out;

1955

goto out;

1956

}

1956

}

1957

1958

static void release_global_resources(void)

1958

static void release_global_resources(void)

1959

{

1959

{

1960

mutex_lock(&spu_lock);

1960

mutex_lock(&spu_lock);

1961

if (!--global_ref) {

1961

if (!--global_ref) {

1962

kfree(cpu_to_cwq);

1962

kfree(cpu_to_cwq);

1963

cpu_to_cwq = NULL;

1963

cpu_to_cwq = NULL;

1964

1965

kfree(cpu_to_mau);

1965

kfree(cpu_to_mau);

1966

cpu_to_mau = NULL;

1966

cpu_to_mau = NULL;

1967

1968

queue_cache_destroy();

1968

queue_cache_destroy();

1969

n2_spu_hvapi_unregister();

1969

n2_spu_hvapi_unregister();

1970

}

1970

}

1971

mutex_unlock(&spu_lock);

1971

mutex_unlock(&spu_lock);

1972

}

1972

}

1973

1974

static struct n2_crypto * __devinit alloc_n2cp(void)

1974

static struct n2_crypto * __devinit alloc_n2cp(void)

1975

{

1975

{

1976

struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);

1976

struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);

1977

1978

if (np)

1978

if (np)

1979

INIT_LIST_HEAD(&np->cwq_list);

1979

INIT_LIST_HEAD(&np->cwq_list);

1980

1981

return np;

1981

return np;

1982

}

1982

}

1983

1984

static void free_n2cp(struct n2_crypto *np)

1984

static void free_n2cp(struct n2_crypto *np)

1985

{

1985

{

1986

if (np->cwq_info.ino_table) {

1986

if (np->cwq_info.ino_table) {

1987

kfree(np->cwq_info.ino_table);

1987

kfree(np->cwq_info.ino_table);

1988

np->cwq_info.ino_table = NULL;

1988

np->cwq_info.ino_table = NULL;

1989

}

1989

}

1990

1991

kfree(np);

1991

kfree(np);

1992

}

1992

}

1993

1994

static void __devinit n2_spu_driver_version(void)

1994

static void __devinit n2_spu_driver_version(void)

1995

{

1995

{

1996

static int n2_spu_version_printed;

1996

static int n2_spu_version_printed;

1997

1998

if (n2_spu_version_printed++ == 0)

1998

if (n2_spu_version_printed++ == 0)

1999

pr_info("%s", version);

1999

pr_info("%s", version);

2000

}

2000

}

2001

2002

static int __devinit n2_crypto_probe(struct platform_device *dev)

2002

static int __devinit n2_crypto_probe(struct platform_device *dev)

2003

{

2003

{

2004

struct mdesc_handle *mdesc;

2004

struct mdesc_handle *mdesc;

2005

const char *full_name;

2005

const char *full_name;

2006

struct n2_crypto *np;

2006

struct n2_crypto *np;

2007

int err;

2007

int err;

2008

2009

n2_spu_driver_version();

2009

n2_spu_driver_version();

2010

2011

full_name = dev->dev.of_node->full_name;

2011

full_name = dev->dev.of_node->full_name;

2012

pr_info("Found N2CP at %s\n", full_name);

2012

pr_info("Found N2CP at %s\n", full_name);

2013

2014

np = alloc_n2cp();

2014

np = alloc_n2cp();

2015

if (!np) {

2015

if (!np) {

2016

dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",

2016

dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",

2017

full_name);

2017

full_name);

2018

return -ENOMEM;

2018

return -ENOMEM;

2019

}

2019

}

2020

2021

err = grab_global_resources();

2021

err = grab_global_resources();

2022

if (err) {

2022

if (err) {

2023

dev_err(&dev->dev, "%s: Unable to grab "

2023

dev_err(&dev->dev, "%s: Unable to grab "

2024

"global resources.\n", full_name);

2024

"global resources.\n", full_name);

2025

goto out_free_n2cp;

2025

goto out_free_n2cp;

2026

}

2026

}

2027

2028

mdesc = mdesc_grab();

2028

mdesc = mdesc_grab();

2029

2030

if (!mdesc) {

2030

if (!mdesc) {

2031

dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",

2031

dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",

2032

full_name);

2032

full_name);

2033

err = -ENODEV;

2033

err = -ENODEV;

2034

goto out_free_global;

2034

goto out_free_global;

2035

}

2035

}

2036

err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");

2036

err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");

2037

if (err) {

2037

if (err) {

2038

dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",

2038

dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",

2039

full_name);

2039

full_name);

2040

mdesc_release(mdesc);

2040

mdesc_release(mdesc);

2041

goto out_free_global;

2041

goto out_free_global;

2042

}

2042

}

2043

2044

err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,

2044

err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,

2045

"cwq", HV_NCS_QTYPE_CWQ, cwq_intr,

2045

"cwq", HV_NCS_QTYPE_CWQ, cwq_intr,

2046

cpu_to_cwq);

2046

cpu_to_cwq);

2047

mdesc_release(mdesc);

2047

mdesc_release(mdesc);

2048

2049

if (err) {

2049

if (err) {

2050

dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",

2050

dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",

2051

full_name);

2051

full_name);

2052

goto out_free_global;

2052

goto out_free_global;

2053

}

2053

}

2054

2055

err = n2_register_algs();

2055

err = n2_register_algs();

2056

if (err) {

2056

if (err) {

2057

dev_err(&dev->dev, "%s: Unable to register algorithms.\n",

2057

dev_err(&dev->dev, "%s: Unable to register algorithms.\n",

2058

full_name);

2058

full_name);

2059

goto out_free_spu_list;

2059

goto out_free_spu_list;

2060

}

2060

}

2061

2062

dev_set_drvdata(&dev->dev, np);

2062

dev_set_drvdata(&dev->dev, np);

2063

2064

return 0;

2064

return 0;

2065

2066

out_free_spu_list:

2066

out_free_spu_list:

2067

spu_list_destroy(&np->cwq_list);

2067

spu_list_destroy(&np->cwq_list);

2068

2069

out_free_global:

2069

out_free_global:

2070

release_global_resources();

2070

release_global_resources();

2071

2072

out_free_n2cp:

2072

out_free_n2cp:

2073

free_n2cp(np);

2073

free_n2cp(np);

2074

2075

return err;

2075

return err;

2076

}

2076

}

2077

2078

static int __devexit n2_crypto_remove(struct platform_device *dev)

2078

static int __devexit n2_crypto_remove(struct platform_device *dev)

2079

{

2079

{

2080

struct n2_crypto *np = dev_get_drvdata(&dev->dev);

2080

struct n2_crypto *np = dev_get_drvdata(&dev->dev);

2081

2082

n2_unregister_algs();

2082

n2_unregister_algs();

2083

2084

spu_list_destroy(&np->cwq_list);

2084

spu_list_destroy(&np->cwq_list);

2085

2086

release_global_resources();

2086

release_global_resources();

2087

2088

free_n2cp(np);

2088

free_n2cp(np);

2089

2090

return 0;

2090

return 0;

2091

}

2091

}

2092

2093

static struct n2_mau * __devinit alloc_ncp(void)

2093

static struct n2_mau * __devinit alloc_ncp(void)

2094

{

2094

{

2095

struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);

2095

struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);

2096

2097

if (mp)

2097

if (mp)

2098

INIT_LIST_HEAD(&mp->mau_list);

2098

INIT_LIST_HEAD(&mp->mau_list);

2099

2100

return mp;

2100

return mp;

2101

}

2101

}

2102

2103

static void free_ncp(struct n2_mau *mp)

2103

static void free_ncp(struct n2_mau *mp)

2104

{

2104

{

2105

if (mp->mau_info.ino_table) {

2105

if (mp->mau_info.ino_table) {

2106

kfree(mp->mau_info.ino_table);

2106

kfree(mp->mau_info.ino_table);

2107

mp->mau_info.ino_table = NULL;

2107

mp->mau_info.ino_table = NULL;

2108

}

2108

}

2109

2110

kfree(mp);

2110

kfree(mp);

2111

}

2111

}

2112

2113

static int __devinit n2_mau_probe(struct platform_device *dev)

2113

static int __devinit n2_mau_probe(struct platform_device *dev)

2114

{

2114

{

2115

struct mdesc_handle *mdesc;

2115

struct mdesc_handle *mdesc;

2116

const char *full_name;

2116

const char *full_name;

2117

struct n2_mau *mp;

2117

struct n2_mau *mp;

2118

int err;

2118

int err;

2119

2120

n2_spu_driver_version();

2120

n2_spu_driver_version();

2121

2122

full_name = dev->dev.of_node->full_name;

2122

full_name = dev->dev.of_node->full_name;

2123

pr_info("Found NCP at %s\n", full_name);

2123

pr_info("Found NCP at %s\n", full_name);

2124

2125

mp = alloc_ncp();

2125

mp = alloc_ncp();

2126

if (!mp) {

2126

if (!mp) {

2127

dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",

2127

dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",

2128

full_name);

2128

full_name);

2129

return -ENOMEM;

2129

return -ENOMEM;

2130

}

2130

}

2131

2132

err = grab_global_resources();

2132

err = grab_global_resources();

2133

if (err) {

2133

if (err) {

2134

dev_err(&dev->dev, "%s: Unable to grab "

2134

dev_err(&dev->dev, "%s: Unable to grab "

2135

"global resources.\n", full_name);

2135

"global resources.\n", full_name);

2136

goto out_free_ncp;

2136

goto out_free_ncp;

2137

}

2137

}

2138

2139

mdesc = mdesc_grab();

2139

mdesc = mdesc_grab();

2140

2141

if (!mdesc) {

2141

if (!mdesc) {

2142

dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",

2142

dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",

2143

full_name);

2143

full_name);

2144

err = -ENODEV;

2144

err = -ENODEV;

2145

goto out_free_global;

2145

goto out_free_global;

2146

}

2146

}

2147

2148

err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");

2148

err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");

2149

if (err) {

2149

if (err) {

2150

dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",

2150

dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",

2151

full_name);

2151

full_name);

2152

mdesc_release(mdesc);

2152

mdesc_release(mdesc);

2153

goto out_free_global;

2153

goto out_free_global;

2154

}

2154

}

2155

2156

err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,

2156

err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,

2157

"mau", HV_NCS_QTYPE_MAU, mau_intr,

2157

"mau", HV_NCS_QTYPE_MAU, mau_intr,

2158

cpu_to_mau);

2158

cpu_to_mau);

2159

mdesc_release(mdesc);

2159

mdesc_release(mdesc);

2160

2161

if (err) {

2161

if (err) {

2162

dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",

2162

dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",

2163

full_name);

2163

full_name);

2164

goto out_free_global;

2164

goto out_free_global;

2165

}

2165

}

2166

2167

dev_set_drvdata(&dev->dev, mp);

2167

dev_set_drvdata(&dev->dev, mp);

2168

2169

return 0;

2169

return 0;

2170

2171

out_free_global:

2171

out_free_global:

2172

release_global_resources();

2172

release_global_resources();

2173

2174

out_free_ncp:

2174

out_free_ncp:

2175

free_ncp(mp);

2175

free_ncp(mp);

2176

2177

return err;

2177

return err;

2178

}

2178

}

2179

2180

static int __devexit n2_mau_remove(struct platform_device *dev)

2180

static int __devexit n2_mau_remove(struct platform_device *dev)

2181

{

2181

{

2182

struct n2_mau *mp = dev_get_drvdata(&dev->dev);

2182

struct n2_mau *mp = dev_get_drvdata(&dev->dev);

2183

2184

spu_list_destroy(&mp->mau_list);

2184

spu_list_destroy(&mp->mau_list);

2185

2186

release_global_resources();

2186

release_global_resources();

2187

2188

free_ncp(mp);

2188

free_ncp(mp);

2189

2190

return 0;

2190

return 0;

2191

}

2191

}

2192

2193

static struct of_device_id n2_crypto_match[] = {

2193

static struct of_device_id n2_crypto_match[] = {

2194

{

2194

{

2195

.name = "n2cp",

2195

.name = "n2cp",

2196

.compatible = "SUNW,n2-cwq",

2196

.compatible = "SUNW,n2-cwq",

2197

},

2197

},

2198

{

2198

{

2199

.name = "n2cp",

2199

.name = "n2cp",

2200

.compatible = "SUNW,vf-cwq",

2200

.compatible = "SUNW,vf-cwq",

2201

},

2201

},

2202

{

2202

{

2203

.name = "n2cp",

2203

.name = "n2cp",

2204

.compatible = "SUNW,kt-cwq",

2204

.compatible = "SUNW,kt-cwq",

2205

},

2205

},

2206

{},

2206

{},

2207

};

2207

};

2208

2209

MODULE_DEVICE_TABLE(of, n2_crypto_match);

2209

MODULE_DEVICE_TABLE(of, n2_crypto_match);

2210

2211

static struct platform_driver n2_crypto_driver = {

2211

static struct platform_driver n2_crypto_driver = {

2212

.driver = {

2212

.driver = {

2213

.name = "n2cp",

2213

.name = "n2cp",

2214

.owner = THIS_MODULE,

2214

.owner = THIS_MODULE,

2215

.of_match_table = n2_crypto_match,

2215

.of_match_table = n2_crypto_match,

2216

},

2216

},

2217

.probe = n2_crypto_probe,

2217

.probe = n2_crypto_probe,

2218

.remove = __devexit_p(n2_crypto_remove),

2218

.remove = __devexit_p(n2_crypto_remove),

2219

};

2219

};

2220

2221

static struct of_device_id n2_mau_match[] = {

2221

static struct of_device_id n2_mau_match[] = {

2222

{

2222

{

2223

.name = "ncp",

2223

.name = "ncp",

2224

.compatible = "SUNW,n2-mau",

2224

.compatible = "SUNW,n2-mau",

2225

},

2225

},

2226

{

2226

{

2227

.name = "ncp",

2227

.name = "ncp",

2228

.compatible = "SUNW,vf-mau",

2228

.compatible = "SUNW,vf-mau",

2229

},

2229

},

2230

{

2230

{

2231

.name = "ncp",

2231

.name = "ncp",

2232

.compatible = "SUNW,kt-mau",

2232

.compatible = "SUNW,kt-mau",

2233

},

2233

},

2234

{},

2234

{},

2235

};

2235

};

2236

2237

MODULE_DEVICE_TABLE(of, n2_mau_match);

2237

MODULE_DEVICE_TABLE(of, n2_mau_match);

2238

2239

static struct platform_driver n2_mau_driver = {

2239

static struct platform_driver n2_mau_driver = {

2240

.driver = {

2240

.driver = {

2241

.name = "ncp",

2241

.name = "ncp",

2242

.owner = THIS_MODULE,

2242

.owner = THIS_MODULE,

2243

.of_match_table = n2_mau_match,

2243

.of_match_table = n2_mau_match,

2244

},

2244

},

2245

.probe = n2_mau_probe,

2245

.probe = n2_mau_probe,

2246

.remove = __devexit_p(n2_mau_remove),

2246

.remove = __devexit_p(n2_mau_remove),

2247

};

2247

};

2248

2249

static int __init n2_init(void)

2249

static int __init n2_init(void)

2250

{

2250

{

2251

int err = platform_driver_register(&n2_crypto_driver);

2251

int err = platform_driver_register(&n2_crypto_driver);

2252

2253

if (!err) {

2253

if (!err) {

2254

err = platform_driver_register(&n2_mau_driver);

2254

err = platform_driver_register(&n2_mau_driver);

2255

if (err)

2255

if (err)

2256

platform_driver_unregister(&n2_crypto_driver);

2256

platform_driver_unregister(&n2_crypto_driver);

2257

}

2257

}

2258

return err;

2258

return err;

2259

}

2259

}

2260

2261

static void __exit n2_exit(void)

2261

static void __exit n2_exit(void)

2262

{

2262

{

2263

platform_driver_unregister(&n2_mau_driver);

2263

platform_driver_unregister(&n2_mau_driver);

2264

platform_driver_unregister(&n2_crypto_driver);

2264

platform_driver_unregister(&n2_crypto_driver);

2265

}

2265

}

2266

2267

module_init(n2_init);

2267

module_init(n2_init);

GITLAB

crypto: n2 - Fix a get/put_cpu() imbalance

 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
  *
  * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/cpumask.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/crypto.h>
 #include <crypto/md5.h>
 #include <crypto/sha.h>
 #include <crypto/aes.h>
 #include <crypto/des.h>
 #include <linux/mutex.h>
 #include <linux/delay.h>
 #include <linux/sched.h>
 #include <crypto/internal/hash.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/algapi.h>
 #include <asm/hypervisor.h>
 #include <asm/mdesc.h>
 #include "n2_core.h"
 #define DRV_MODULE_NAME		"n2_crypto"
 #define DRV_MODULE_VERSION	"0.2"
 #define DRV_MODULE_RELDATE	"July 28, 2011"
 static char version[] __devinitdata =
 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
 MODULE_DESCRIPTION("Niagara2 Crypto driver");
 MODULE_LICENSE("GPL");
 MODULE_VERSION(DRV_MODULE_VERSION);
 #define N2_CRA_PRIORITY		300
 static DEFINE_MUTEX(spu_lock);
 struct spu_queue {
 	cpumask_t		sharing;
 	unsigned long		qhandle;
 	spinlock_t		lock;
 	u8			q_type;
 	void			*q;
 	unsigned long		head;
 	unsigned long		tail;
 	struct list_head	jobs;
 	unsigned long		devino;
 	char			irq_name[32];
 	unsigned int		irq;
 	struct list_head	list;
 };
 static struct spu_queue **cpu_to_cwq;
 static struct spu_queue **cpu_to_mau;
 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
 {
 	if (q->q_type == HV_NCS_QTYPE_MAU) {
 		off += MAU_ENTRY_SIZE;
 		if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
 			off = 0;
 	} else {
 		off += CWQ_ENTRY_SIZE;
 		if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
 			off = 0;
 	}
 	return off;
 }
 struct n2_request_common {
 	struct list_head	entry;
 	unsigned int		offset;
 };
 #define OFFSET_NOT_RUNNING	(~(unsigned int)0)
 /* An async job request records the final tail value it used in
  * n2_request_common->offset, test to see if that offset is in
  * the range old_head, new_head, inclusive.
  */
 static inline bool job_finished(struct spu_queue *q, unsigned int offset,
 				unsigned long old_head, unsigned long new_head)
 {
 	if (old_head <= new_head) {
 		if (offset > old_head && offset <= new_head)
 			return true;
 	} else {
 		if (offset > old_head || offset <= new_head)
 			return true;
 	}
 	return false;
 }
 /* When the HEAD marker is unequal to the actual HEAD, we get
  * a virtual device INO interrupt.  We should process the
  * completed CWQ entries and adjust the HEAD marker to clear
  * the IRQ.
  */
 static irqreturn_t cwq_intr(int irq, void *dev_id)
 {
 	unsigned long off, new_head, hv_ret;
 	struct spu_queue *q = dev_id;
 	pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
 	       smp_processor_id(), q->qhandle);
 	spin_lock(&q->lock);
 	hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
 	pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
 	       smp_processor_id(), new_head, hv_ret);
 	for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
 		/* XXX ... XXX */
 	}
 	hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
 	if (hv_ret == HV_EOK)
 		q->head = new_head;
 	spin_unlock(&q->lock);
 	return IRQ_HANDLED;
 }
 static irqreturn_t mau_intr(int irq, void *dev_id)
 {
 	struct spu_queue *q = dev_id;
 	unsigned long head, hv_ret;
 	spin_lock(&q->lock);
 	pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
 	       smp_processor_id(), q->qhandle);
 	hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
 	pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
 	       smp_processor_id(), head, hv_ret);
 	sun4v_ncs_sethead_marker(q->qhandle, head);
 	spin_unlock(&q->lock);
 	return IRQ_HANDLED;
 }
 static void *spu_queue_next(struct spu_queue *q, void *cur)
 {
 	return q->q + spu_next_offset(q, cur - q->q);
 }
 static int spu_queue_num_free(struct spu_queue *q)
 {
 	unsigned long head = q->head;
 	unsigned long tail = q->tail;
 	unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
 	unsigned long diff;
 	if (head > tail)
 		diff = head - tail;
 	else
 		diff = (end - tail) + head;
 	return (diff / CWQ_ENTRY_SIZE) - 1;
 }
 static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
 {
 	int avail = spu_queue_num_free(q);
 	if (avail >= num_entries)
 		return q->q + q->tail;
 	return NULL;
 }
 static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
 {
 	unsigned long hv_ret, new_tail;
 	new_tail = spu_next_offset(q, last - q->q);
 	hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
 	if (hv_ret == HV_EOK)
 		q->tail = new_tail;
 	return hv_ret;
 }
 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
 			     int enc_type, int auth_type,
 			     unsigned int hash_len,
 			     bool sfas, bool sob, bool eob, bool encrypt,
 			     int opcode)
 {
 	u64 word = (len - 1) & CONTROL_LEN;
 	word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
 	word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
 	word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
 	if (sfas)
 		word |= CONTROL_STORE_FINAL_AUTH_STATE;
 	if (sob)
 		word |= CONTROL_START_OF_BLOCK;
 	if (eob)
 		word |= CONTROL_END_OF_BLOCK;
 	if (encrypt)
 		word |= CONTROL_ENCRYPT;
 	if (hmac_key_len)
 		word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
 	if (hash_len)
 		word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
 	return word;
 }
 #if 0
 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
 {
 	if (this_len >= 64 ||
 	    qp->head != qp->tail)
 		return true;
 	return false;
 }
 #endif
 struct n2_ahash_alg {
 	struct list_head	entry;
 	const char		*hash_zero;
 	const u32		*hash_init;
 	u8			hw_op_hashsz;
 	u8			digest_size;
 	u8			auth_type;
 	u8			hmac_type;
 	struct ahash_alg	alg;
 };
 static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
 {
 	struct crypto_alg *alg = tfm->__crt_alg;
 	struct ahash_alg *ahash_alg;
 	ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 	return container_of(ahash_alg, struct n2_ahash_alg, alg);
 }
 struct n2_hmac_alg {
 	const char		*child_alg;
 	struct n2_ahash_alg	derived;
 };
 static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
 {
 	struct crypto_alg *alg = tfm->__crt_alg;
 	struct ahash_alg *ahash_alg;
 	ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 	return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
 }
 struct n2_hash_ctx {
 	struct crypto_ahash		*fallback_tfm;
 };
 #define N2_HASH_KEY_MAX			32 /* HW limit for all HMAC requests */
 struct n2_hmac_ctx {
 	struct n2_hash_ctx		base;
 	struct crypto_shash		*child_shash;
 	int				hash_key_len;
 	unsigned char			hash_key[N2_HASH_KEY_MAX];
 };
 struct n2_hash_req_ctx {
 	union {
 		struct md5_state	md5;
 		struct sha1_state	sha1;
 		struct sha256_state	sha256;
 	} u;
 	struct ahash_request		fallback_req;
 };
 static int n2_hash_async_init(struct ahash_request *req)
 {
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_ahash_init(&rctx->fallback_req);
 }
 static int n2_hash_async_update(struct ahash_request *req)
 {
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	rctx->fallback_req.nbytes = req->nbytes;
 	rctx->fallback_req.src = req->src;
 	return crypto_ahash_update(&rctx->fallback_req);
 }
 static int n2_hash_async_final(struct ahash_request *req)
 {
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	rctx->fallback_req.result = req->result;
 	return crypto_ahash_final(&rctx->fallback_req);
 }
 static int n2_hash_async_finup(struct ahash_request *req)
 {
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	rctx->fallback_req.nbytes = req->nbytes;
 	rctx->fallback_req.src = req->src;
 	rctx->fallback_req.result = req->result;
 	return crypto_ahash_finup(&rctx->fallback_req);
 }
 static int n2_hash_cra_init(struct crypto_tfm *tfm)
 {
 	const char *fallback_driver_name = tfm->__crt_alg->cra_name;
 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	struct crypto_ahash *fallback_tfm;
 	int err;
 	fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 					  CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(fallback_tfm)) {
 		pr_warning("Fallback driver '%s' could not be loaded!\n",
 			   fallback_driver_name);
 		err = PTR_ERR(fallback_tfm);
 		goto out;
 	}
 	crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 					 crypto_ahash_reqsize(fallback_tfm)));
 	ctx->fallback_tfm = fallback_tfm;
 	return 0;
 out:
 	return err;
 }
 static void n2_hash_cra_exit(struct crypto_tfm *tfm)
 {
 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	crypto_free_ahash(ctx->fallback_tfm);
 }
 static int n2_hmac_cra_init(struct crypto_tfm *tfm)
 {
 	const char *fallback_driver_name = tfm->__crt_alg->cra_name;
 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 	struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
 	struct crypto_ahash *fallback_tfm;
 	struct crypto_shash *child_shash;
 	int err;
 	fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 					  CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(fallback_tfm)) {
 		pr_warning("Fallback driver '%s' could not be loaded!\n",
 			   fallback_driver_name);
 		err = PTR_ERR(fallback_tfm);
 		goto out;
 	}
 	child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
 	if (IS_ERR(child_shash)) {
 		pr_warning("Child shash '%s' could not be loaded!\n",
 			   n2alg->child_alg);
 		err = PTR_ERR(child_shash);
 		goto out_free_fallback;
 	}
 	crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 					 crypto_ahash_reqsize(fallback_tfm)));
 	ctx->child_shash = child_shash;
 	ctx->base.fallback_tfm = fallback_tfm;
 	return 0;
 out_free_fallback:
 	crypto_free_ahash(fallback_tfm);
 out:
 	return err;
 }
 static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
 {
 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 	crypto_free_ahash(ctx->base.fallback_tfm);
 	crypto_free_shash(ctx->child_shash);
 }
 static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
 				unsigned int keylen)
 {
 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct crypto_shash *child_shash = ctx->child_shash;
 	struct crypto_ahash *fallback_tfm;
 	struct {
 		struct shash_desc shash;
 		char ctx[crypto_shash_descsize(child_shash)];
 	} desc;
 	int err, bs, ds;
 	fallback_tfm = ctx->base.fallback_tfm;
 	err = crypto_ahash_setkey(fallback_tfm, key, keylen);
 	if (err)
 		return err;
 	desc.shash.tfm = child_shash;
 	desc.shash.flags = crypto_ahash_get_flags(tfm) &
 		CRYPTO_TFM_REQ_MAY_SLEEP;
 	bs = crypto_shash_blocksize(child_shash);
 	ds = crypto_shash_digestsize(child_shash);
 	BUG_ON(ds > N2_HASH_KEY_MAX);
 	if (keylen > bs) {
 		err = crypto_shash_digest(&desc.shash, key, keylen,
 					  ctx->hash_key);
 		if (err)
 			return err;
 		keylen = ds;
 	} else if (keylen <= N2_HASH_KEY_MAX)
 		memcpy(ctx->hash_key, key, keylen);
 	ctx->hash_key_len = keylen;
 	return err;
 }
 static unsigned long wait_for_tail(struct spu_queue *qp)
 {
 	unsigned long head, hv_ret;
 	do {
 		hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
 		if (hv_ret != HV_EOK) {
 			pr_err("Hypervisor error on gethead\n");
 			break;
 		}
 		if (head == qp->tail) {
 			qp->head = head;
 			break;
 		}
 	} while (1);
 	return hv_ret;
 }
 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
 					      struct cwq_initial_entry *ent)
 {
 	unsigned long hv_ret = spu_queue_submit(qp, ent);
 	if (hv_ret == HV_EOK)
 		hv_ret = wait_for_tail(qp);
 	return hv_ret;
 }
 static int n2_do_async_digest(struct ahash_request *req,
 			      unsigned int auth_type, unsigned int digest_size,
 			      unsigned int result_size, void *hash_loc,
 			      unsigned long auth_key, unsigned int auth_key_len)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct cwq_initial_entry *ent;
 	struct crypto_hash_walk walk;
 	struct spu_queue *qp;
 	unsigned long flags;
 	int err = -ENODEV;
 	int nbytes, cpu;
 	/* The total effective length of the operation may not
 	 * exceed 2^16.
 	 */
 	if (unlikely(req->nbytes > (1 << 16))) {
 		struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 		struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 		ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 		rctx->fallback_req.base.flags =
 			req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 		rctx->fallback_req.nbytes = req->nbytes;
 		rctx->fallback_req.src = req->src;
 		rctx->fallback_req.result = req->result;
 		return crypto_ahash_digest(&rctx->fallback_req);
 	}
 	nbytes = crypto_hash_walk_first(req, &walk);
 	cpu = get_cpu();
 	qp = cpu_to_cwq[cpu];
 	if (!qp)
 		goto out;
 	spin_lock_irqsave(&qp->lock, flags);
 	/* XXX can do better, improve this later by doing a by-hand scatterlist
 	 * XXX walk, etc.
 	 */
 	ent = qp->q + qp->tail;
 	ent->control = control_word_base(nbytes, auth_key_len, 0,
 					 auth_type, digest_size,
 					 false, true, false, false,
 					 OPCODE_INPLACE_BIT |
 					 OPCODE_AUTH_MAC);
 	ent->src_addr = __pa(walk.data);
 	ent->auth_key_addr = auth_key;
 	ent->auth_iv_addr = __pa(hash_loc);
 	ent->final_auth_state_addr = 0UL;
 	ent->enc_key_addr = 0UL;
 	ent->enc_iv_addr = 0UL;
 	ent->dest_addr = __pa(hash_loc);
 	nbytes = crypto_hash_walk_done(&walk, 0);
 	while (nbytes > 0) {
 		ent = spu_queue_next(qp, ent);
 		ent->control = (nbytes - 1);
 		ent->src_addr = __pa(walk.data);
 		ent->auth_key_addr = 0UL;
 		ent->auth_iv_addr = 0UL;
 		ent->final_auth_state_addr = 0UL;
 		ent->enc_key_addr = 0UL;
 		ent->enc_iv_addr = 0UL;
 		ent->dest_addr = 0UL;
 		nbytes = crypto_hash_walk_done(&walk, 0);
 	}
 	ent->control |= CONTROL_END_OF_BLOCK;
 	if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
 		err = -EINVAL;
 	else
 		err = 0;
 	spin_unlock_irqrestore(&qp->lock, flags);
 	if (!err)
 		memcpy(req->result, hash_loc, result_size);
 out:
 	put_cpu();
 	return err;
 }
 static int n2_hash_async_digest(struct ahash_request *req)
 {
 	struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	int ds;
 	ds = n2alg->digest_size;
 	if (unlikely(req->nbytes == 0)) {
 		memcpy(req->result, n2alg->hash_zero, ds);
 		return 0;
 	}
 	memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
 	return n2_do_async_digest(req, n2alg->auth_type,
 				  n2alg->hw_op_hashsz, ds,
 				  &rctx->u, 0UL, 0);
 }
 static int n2_hmac_async_digest(struct ahash_request *req)
 {
 	struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 	int ds;
 	ds = n2alg->derived.digest_size;
 	if (unlikely(req->nbytes == 0) ||
 	    unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
 		struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 		struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 		ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 		rctx->fallback_req.base.flags =
 			req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 		rctx->fallback_req.nbytes = req->nbytes;
 		rctx->fallback_req.src = req->src;
 		rctx->fallback_req.result = req->result;
 		return crypto_ahash_digest(&rctx->fallback_req);
 	}
 	memcpy(&rctx->u, n2alg->derived.hash_init,
 	       n2alg->derived.hw_op_hashsz);
 	return n2_do_async_digest(req, n2alg->derived.hmac_type,
 				  n2alg->derived.hw_op_hashsz, ds,
 				  &rctx->u,
 				  __pa(&ctx->hash_key),
 				  ctx->hash_key_len);
 }
 struct n2_cipher_context {
 	int			key_len;
 	int			enc_type;
 	union {
 		u8		aes[AES_MAX_KEY_SIZE];
 		u8		des[DES_KEY_SIZE];
 		u8		des3[3 * DES_KEY_SIZE];
 		u8		arc4[258]; /* S-box, X, Y */
 	} key;
 };
 #define N2_CHUNK_ARR_LEN	16
 struct n2_crypto_chunk {
 	struct list_head	entry;
 	unsigned long		iv_paddr : 44;
 	unsigned long		arr_len : 20;
 	unsigned long		dest_paddr;
 	unsigned long		dest_final;
 	struct {
 		unsigned long	src_paddr : 44;
 		unsigned long	src_len : 20;
 	} arr[N2_CHUNK_ARR_LEN];
 };
 struct n2_request_context {
 	struct ablkcipher_walk	walk;
 	struct list_head	chunk_list;
 	struct n2_crypto_chunk	chunk;
 	u8			temp_iv[16];
 };
 /* The SPU allows some level of flexibility for partial cipher blocks
  * being specified in a descriptor.
  *
  * It merely requires that every descriptor's length field is at least
  * as large as the cipher block size.  This means that a cipher block
  * can span at most 2 descriptors.  However, this does not allow a
  * partial block to span into the final descriptor as that would
  * violate the rule (since every descriptor's length must be at lest
  * the block size).  So, for example, assuming an 8 byte block size:
  *
  *	0xe --> 0xa --> 0x8
  *
  * is a valid length sequence, whereas:
  *
  *	0xe --> 0xb --> 0x7
  *
  * is not a valid sequence.
  */
 struct n2_cipher_alg {
 	struct list_head	entry;
 	u8			enc_type;
 	struct crypto_alg	alg;
 };
 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
 {
 	struct crypto_alg *alg = tfm->__crt_alg;
 	return container_of(alg, struct n2_cipher_alg, alg);
 }
 struct n2_cipher_request_context {
 	struct ablkcipher_walk	walk;
 };
 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 			 unsigned int keylen)
 {
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 	ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
 	switch (keylen) {
 	case AES_KEYSIZE_128:
 		ctx->enc_type |= ENC_TYPE_ALG_AES128;
 		break;
 	case AES_KEYSIZE_192:
 		ctx->enc_type |= ENC_TYPE_ALG_AES192;
 		break;
 	case AES_KEYSIZE_256:
 		ctx->enc_type |= ENC_TYPE_ALG_AES256;
 		break;
 	default:
 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	ctx->key_len = keylen;
 	memcpy(ctx->key.aes, key, keylen);
 	return 0;
 }
 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 			 unsigned int keylen)
 {
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 	u32 tmp[DES_EXPKEY_WORDS];
 	int err;
 	ctx->enc_type = n2alg->enc_type;
 	if (keylen != DES_KEY_SIZE) {
 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	err = des_ekey(tmp, key);
 	if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 		tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 		return -EINVAL;
 	}
 	ctx->key_len = keylen;
 	memcpy(ctx->key.des, key, keylen);
 	return 0;
 }
 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 			  unsigned int keylen)
 {
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 	ctx->enc_type = n2alg->enc_type;
 	if (keylen != (3 * DES_KEY_SIZE)) {
 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	ctx->key_len = keylen;
 	memcpy(ctx->key.des3, key, keylen);
 	return 0;
 }
 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 			  unsigned int keylen)
 {
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 	u8 *s = ctx->key.arc4;
 	u8 *x = s + 256;
 	u8 *y = x + 1;
 	int i, j, k;
 	ctx->enc_type = n2alg->enc_type;
 	j = k = 0;
 	*x = 0;
 	*y = 0;
 	for (i = 0; i < 256; i++)
 		s[i] = i;
 	for (i = 0; i < 256; i++) {
 		u8 a = s[i];
 		j = (j + key[k] + a) & 0xff;
 		s[i] = s[j];
 		s[j] = a;
 		if (++k >= keylen)
 			k = 0;
 	}
 	return 0;
 }
 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
 {
 	int this_len = nbytes;
 	this_len -= (nbytes & (block_size - 1));
 	return this_len > (1 << 16) ? (1 << 16) : this_len;
 }
 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
 			    struct spu_queue *qp, bool encrypt)
 {
 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 	struct cwq_initial_entry *ent;
 	bool in_place;
 	int i;
 	ent = spu_queue_alloc(qp, cp->arr_len);
 	if (!ent) {
 		pr_info("queue_alloc() of %d fails\n",
 			cp->arr_len);
 		return -EBUSY;
 	}
 	in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
 	ent->control = control_word_base(cp->arr[0].src_len,
 					 0, ctx->enc_type, 0, 0,
 					 false, true, false, encrypt,
 					 OPCODE_ENCRYPT |
 					 (in_place ? OPCODE_INPLACE_BIT : 0));
 	ent->src_addr = cp->arr[0].src_paddr;
 	ent->auth_key_addr = 0UL;
 	ent->auth_iv_addr = 0UL;
 	ent->final_auth_state_addr = 0UL;
 	ent->enc_key_addr = __pa(&ctx->key);
 	ent->enc_iv_addr = cp->iv_paddr;
 	ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
 	for (i = 1; i < cp->arr_len; i++) {
 		ent = spu_queue_next(qp, ent);
 		ent->control = cp->arr[i].src_len - 1;
 		ent->src_addr = cp->arr[i].src_paddr;
 		ent->auth_key_addr = 0UL;
 		ent->auth_iv_addr = 0UL;
 		ent->final_auth_state_addr = 0UL;
 		ent->enc_key_addr = 0UL;
 		ent->enc_iv_addr = 0UL;
 		ent->dest_addr = 0UL;
 	}
 	ent->control |= CONTROL_END_OF_BLOCK;
 	return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
 }
 static int n2_compute_chunks(struct ablkcipher_request *req)
 {
 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 	struct ablkcipher_walk *walk = &rctx->walk;
 	struct n2_crypto_chunk *chunk;
 	unsigned long dest_prev;
 	unsigned int tot_len;
 	bool prev_in_place;
 	int err, nbytes;
 	ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
 	err = ablkcipher_walk_phys(req, walk);
 	if (err)
 		return err;
 	INIT_LIST_HEAD(&rctx->chunk_list);
 	chunk = &rctx->chunk;
 	INIT_LIST_HEAD(&chunk->entry);
 	chunk->iv_paddr = 0UL;
 	chunk->arr_len = 0;
 	chunk->dest_paddr = 0UL;
 	prev_in_place = false;
 	dest_prev = ~0UL;
 	tot_len = 0;
 	while ((nbytes = walk->nbytes) != 0) {
 		unsigned long dest_paddr, src_paddr;
 		bool in_place;
 		int this_len;
 		src_paddr = (page_to_phys(walk->src.page) +
 			     walk->src.offset);
 		dest_paddr = (page_to_phys(walk->dst.page) +
 			      walk->dst.offset);
 		in_place = (src_paddr == dest_paddr);
 		this_len = cipher_descriptor_len(nbytes, walk->blocksize);
 		if (chunk->arr_len != 0) {
 			if (in_place != prev_in_place ||
 			    (!prev_in_place &&
 			     dest_paddr != dest_prev) ||
 			    chunk->arr_len == N2_CHUNK_ARR_LEN ||
 			    tot_len + this_len > (1 << 16)) {
 				chunk->dest_final = dest_prev;
 				list_add_tail(&chunk->entry,
 					      &rctx->chunk_list);
 				chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
 				if (!chunk) {
 					err = -ENOMEM;
 					break;
 				}
 				INIT_LIST_HEAD(&chunk->entry);
 			}
 		}
 		if (chunk->arr_len == 0) {
 			chunk->dest_paddr = dest_paddr;
 			tot_len = 0;
 		}
 		chunk->arr[chunk->arr_len].src_paddr = src_paddr;
 		chunk->arr[chunk->arr_len].src_len = this_len;
 		chunk->arr_len++;
 		dest_prev = dest_paddr + this_len;
 		prev_in_place = in_place;
 		tot_len += this_len;
 		err = ablkcipher_walk_done(req, walk, nbytes - this_len);
 		if (err)
 			break;
 	}
 	if (!err && chunk->arr_len != 0) {
 		chunk->dest_final = dest_prev;
 		list_add_tail(&chunk->entry, &rctx->chunk_list);
 	}
 	return err;
 }
 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
 {
 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 	struct n2_crypto_chunk *c, *tmp;
 	if (final_iv)
 		memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
 	ablkcipher_walk_complete(&rctx->walk);
 	list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 		list_del(&c->entry);
 		if (unlikely(c != &rctx->chunk))
 			kfree(c);
 	}
 }
 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
 {
 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 	struct crypto_tfm *tfm = req->base.tfm;
 	int err = n2_compute_chunks(req);
 	struct n2_crypto_chunk *c, *tmp;
 	unsigned long flags, hv_ret;
 	struct spu_queue *qp;
 	if (err)
 		return err;
 	qp = cpu_to_cwq[get_cpu()];
 	err = -ENODEV;
 	if (!qp)
 		goto out;
 	spin_lock_irqsave(&qp->lock, flags);
 	list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 		err = __n2_crypt_chunk(tfm, c, qp, encrypt);
 		if (err)
 			break;
 		list_del(&c->entry);
 		if (unlikely(c != &rctx->chunk))
 			kfree(c);
 	}
 	if (!err) {
 		hv_ret = wait_for_tail(qp);
 		if (hv_ret != HV_EOK)
 			err = -EINVAL;
 	}
 	spin_unlock_irqrestore(&qp->lock, flags);
+out:
 	put_cpu();
-out:
 	n2_chunk_complete(req, NULL);
 	return err;
 }
 static int n2_encrypt_ecb(struct ablkcipher_request *req)
 {
 	return n2_do_ecb(req, true);
 }
 static int n2_decrypt_ecb(struct ablkcipher_request *req)
 {
 	return n2_do_ecb(req, false);
 }
 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
 {
 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 	struct crypto_tfm *tfm = req->base.tfm;
 	unsigned long flags, hv_ret, iv_paddr;
 	int err = n2_compute_chunks(req);
 	struct n2_crypto_chunk *c, *tmp;
 	struct spu_queue *qp;
 	void *final_iv_addr;
 	final_iv_addr = NULL;
 	if (err)
 		return err;
 	qp = cpu_to_cwq[get_cpu()];
 	err = -ENODEV;
 	if (!qp)
 		goto out;
 	spin_lock_irqsave(&qp->lock, flags);
 	if (encrypt) {
 		iv_paddr = __pa(rctx->walk.iv);
 		list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
 					 entry) {
 			c->iv_paddr = iv_paddr;
 			err = __n2_crypt_chunk(tfm, c, qp, true);
 			if (err)
 				break;
 			iv_paddr = c->dest_final - rctx->walk.blocksize;
 			list_del(&c->entry);
 			if (unlikely(c != &rctx->chunk))
 				kfree(c);
 		}
 		final_iv_addr = __va(iv_paddr);
 	} else {
 		list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
 						 entry) {
 			if (c == &rctx->chunk) {
 				iv_paddr = __pa(rctx->walk.iv);
 			} else {
 				iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
 					    tmp->arr[tmp->arr_len-1].src_len -
 					    rctx->walk.blocksize);
 			}
 			if (!final_iv_addr) {
 				unsigned long pa;
 				pa = (c->arr[c->arr_len-1].src_paddr +
 				      c->arr[c->arr_len-1].src_len -
 				      rctx->walk.blocksize);
 				final_iv_addr = rctx->temp_iv;
 				memcpy(rctx->temp_iv, __va(pa),
 				       rctx->walk.blocksize);
 			}
 			c->iv_paddr = iv_paddr;
 			err = __n2_crypt_chunk(tfm, c, qp, false);
 			if (err)
 				break;
 			list_del(&c->entry);
 			if (unlikely(c != &rctx->chunk))
 				kfree(c);
 		}
 	}
 	if (!err) {
 		hv_ret = wait_for_tail(qp);
 		if (hv_ret != HV_EOK)
 			err = -EINVAL;
 	}
 	spin_unlock_irqrestore(&qp->lock, flags);
+out:
 	put_cpu();
-out:
 	n2_chunk_complete(req, err ? NULL : final_iv_addr);
 	return err;
 }
 static int n2_encrypt_chaining(struct ablkcipher_request *req)
 {
 	return n2_do_chaining(req, true);
 }
 static int n2_decrypt_chaining(struct ablkcipher_request *req)
 {
 	return n2_do_chaining(req, false);
 }
 struct n2_cipher_tmpl {
 	const char		*name;
 	const char		*drv_name;
 	u8			block_size;
 	u8			enc_type;
 	struct ablkcipher_alg	ablkcipher;
 };
 static const struct n2_cipher_tmpl cipher_tmpls[] = {
 	/* ARC4: only ECB is supported (chaining bits ignored) */
 	{	.name		= "ecb(arc4)",
 		.drv_name	= "ecb-arc4",
 		.block_size	= 1,
 		.enc_type	= (ENC_TYPE_ALG_RC4_STREAM |
 				   ENC_TYPE_CHAINING_ECB),
 		.ablkcipher	= {
 			.min_keysize	= 1,
 			.max_keysize	= 256,
 			.setkey		= n2_arc4_setkey,
 			.encrypt	= n2_encrypt_ecb,
 			.decrypt	= n2_decrypt_ecb,
 		},
 	},
 	/* DES: ECB CBC and CFB are supported */
 	{	.name		= "ecb(des)",
 		.drv_name	= "ecb-des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_DES |
 				   ENC_TYPE_CHAINING_ECB),
 		.ablkcipher	= {
 			.min_keysize	= DES_KEY_SIZE,
 			.max_keysize	= DES_KEY_SIZE,
 			.setkey		= n2_des_setkey,
 			.encrypt	= n2_encrypt_ecb,
 			.decrypt	= n2_decrypt_ecb,
 		},
 	},
 	{	.name		= "cbc(des)",
 		.drv_name	= "cbc-des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_DES |
 				   ENC_TYPE_CHAINING_CBC),
 		.ablkcipher	= {
 			.ivsize		= DES_BLOCK_SIZE,
 			.min_keysize	= DES_KEY_SIZE,
 			.max_keysize	= DES_KEY_SIZE,
 			.setkey		= n2_des_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_decrypt_chaining,
 		},
 	},
 	{	.name		= "cfb(des)",
 		.drv_name	= "cfb-des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_DES |
 				   ENC_TYPE_CHAINING_CFB),
 		.ablkcipher	= {
 			.min_keysize	= DES_KEY_SIZE,
 			.max_keysize	= DES_KEY_SIZE,
 			.setkey		= n2_des_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_decrypt_chaining,
 		},
 	},
 	/* 3DES: ECB CBC and CFB are supported */
 	{	.name		= "ecb(des3_ede)",
 		.drv_name	= "ecb-3des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_3DES |
 				   ENC_TYPE_CHAINING_ECB),
 		.ablkcipher	= {
 			.min_keysize	= 3 * DES_KEY_SIZE,
 			.max_keysize	= 3 * DES_KEY_SIZE,
 			.setkey		= n2_3des_setkey,
 			.encrypt	= n2_encrypt_ecb,
 			.decrypt	= n2_decrypt_ecb,
 		},
 	},
 	{	.name		= "cbc(des3_ede)",
 		.drv_name	= "cbc-3des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_3DES |
 				   ENC_TYPE_CHAINING_CBC),
 		.ablkcipher	= {
 			.ivsize		= DES_BLOCK_SIZE,
 			.min_keysize	= 3 * DES_KEY_SIZE,
 			.max_keysize	= 3 * DES_KEY_SIZE,
 			.setkey		= n2_3des_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_decrypt_chaining,
 		},
 	},
 	{	.name		= "cfb(des3_ede)",
 		.drv_name	= "cfb-3des",
 		.block_size	= DES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_3DES |
 				   ENC_TYPE_CHAINING_CFB),
 		.ablkcipher	= {
 			.min_keysize	= 3 * DES_KEY_SIZE,
 			.max_keysize	= 3 * DES_KEY_SIZE,
 			.setkey		= n2_3des_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_decrypt_chaining,
 		},
 	},
 	/* AES: ECB CBC and CTR are supported */
 	{	.name		= "ecb(aes)",
 		.drv_name	= "ecb-aes",
 		.block_size	= AES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_AES128 |
 				   ENC_TYPE_CHAINING_ECB),
 		.ablkcipher	= {
 			.min_keysize	= AES_MIN_KEY_SIZE,
 			.max_keysize	= AES_MAX_KEY_SIZE,
 			.setkey		= n2_aes_setkey,
 			.encrypt	= n2_encrypt_ecb,
 			.decrypt	= n2_decrypt_ecb,
 		},
 	},
 	{	.name		= "cbc(aes)",
 		.drv_name	= "cbc-aes",
 		.block_size	= AES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_AES128 |
 				   ENC_TYPE_CHAINING_CBC),
 		.ablkcipher	= {
 			.ivsize		= AES_BLOCK_SIZE,
 			.min_keysize	= AES_MIN_KEY_SIZE,
 			.max_keysize	= AES_MAX_KEY_SIZE,
 			.setkey		= n2_aes_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_decrypt_chaining,
 		},
 	},
 	{	.name		= "ctr(aes)",
 		.drv_name	= "ctr-aes",
 		.block_size	= AES_BLOCK_SIZE,
 		.enc_type	= (ENC_TYPE_ALG_AES128 |
 				   ENC_TYPE_CHAINING_COUNTER),
 		.ablkcipher	= {
 			.ivsize		= AES_BLOCK_SIZE,
 			.min_keysize	= AES_MIN_KEY_SIZE,
 			.max_keysize	= AES_MAX_KEY_SIZE,
 			.setkey		= n2_aes_setkey,
 			.encrypt	= n2_encrypt_chaining,
 			.decrypt	= n2_encrypt_chaining,
 		},
 	},
 };
 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
 static LIST_HEAD(cipher_algs);
 struct n2_hash_tmpl {
 	const char	*name;
 	const char	*hash_zero;
 	const u32	*hash_init;
 	u8		hw_op_hashsz;
 	u8		digest_size;
 	u8		block_size;
 	u8		auth_type;
 	u8		hmac_type;
 };
 static const char md5_zero[MD5_DIGEST_SIZE] = {
 	0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
 	0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
 };
 static const u32 md5_init[MD5_HASH_WORDS] = {
 	cpu_to_le32(0x67452301),
 	cpu_to_le32(0xefcdab89),
 	cpu_to_le32(0x98badcfe),
 	cpu_to_le32(0x10325476),
 };
 static const char sha1_zero[SHA1_DIGEST_SIZE] = {
 	0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
 	0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
 	0x07, 0x09
 };
 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
 	SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
 };
 static const char sha256_zero[SHA256_DIGEST_SIZE] = {
 	0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
 	0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
 	0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
 	0x1b, 0x78, 0x52, 0xb8, 0x55
 };
 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
 	SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
 	SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
 };
 static const char sha224_zero[SHA224_DIGEST_SIZE] = {
 	0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
 	0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
 	0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
 	0x2f
 };
 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
 	SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
 	SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
 };
 static const struct n2_hash_tmpl hash_tmpls[] = {
 	{ .name		= "md5",
 	  .hash_zero	= md5_zero,
 	  .hash_init	= md5_init,
 	  .auth_type	= AUTH_TYPE_MD5,
 	  .hmac_type	= AUTH_TYPE_HMAC_MD5,
 	  .hw_op_hashsz	= MD5_DIGEST_SIZE,
 	  .digest_size	= MD5_DIGEST_SIZE,
 	  .block_size	= MD5_HMAC_BLOCK_SIZE },
 	{ .name		= "sha1",
 	  .hash_zero	= sha1_zero,
 	  .hash_init	= sha1_init,
 	  .auth_type	= AUTH_TYPE_SHA1,
 	  .hmac_type	= AUTH_TYPE_HMAC_SHA1,
 	  .hw_op_hashsz	= SHA1_DIGEST_SIZE,
 	  .digest_size	= SHA1_DIGEST_SIZE,
 	  .block_size	= SHA1_BLOCK_SIZE },
 	{ .name		= "sha256",
 	  .hash_zero	= sha256_zero,
 	  .hash_init	= sha256_init,
 	  .auth_type	= AUTH_TYPE_SHA256,
 	  .hmac_type	= AUTH_TYPE_HMAC_SHA256,
 	  .hw_op_hashsz	= SHA256_DIGEST_SIZE,
 	  .digest_size	= SHA256_DIGEST_SIZE,
 	  .block_size	= SHA256_BLOCK_SIZE },
 	{ .name		= "sha224",
 	  .hash_zero	= sha224_zero,
 	  .hash_init	= sha224_init,
 	  .auth_type	= AUTH_TYPE_SHA256,
 	  .hmac_type	= AUTH_TYPE_RESERVED,
 	  .hw_op_hashsz	= SHA256_DIGEST_SIZE,
 	  .digest_size	= SHA224_DIGEST_SIZE,
 	  .block_size	= SHA224_BLOCK_SIZE },
 };
 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
 static LIST_HEAD(ahash_algs);
 static LIST_HEAD(hmac_algs);
 static int algs_registered;
 static void __n2_unregister_algs(void)
 {
 	struct n2_cipher_alg *cipher, *cipher_tmp;
 	struct n2_ahash_alg *alg, *alg_tmp;
 	struct n2_hmac_alg *hmac, *hmac_tmp;
 	list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
 		crypto_unregister_alg(&cipher->alg);
 		list_del(&cipher->entry);
 		kfree(cipher);
 	}
 	list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
 		crypto_unregister_ahash(&hmac->derived.alg);
 		list_del(&hmac->derived.entry);
 		kfree(hmac);
 	}
 	list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
 		crypto_unregister_ahash(&alg->alg);
 		list_del(&alg->entry);
 		kfree(alg);
 	}
 }
 static int n2_cipher_cra_init(struct crypto_tfm *tfm)
 {
 	tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
 	return 0;
 }
 static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
 {
 	struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
 	struct crypto_alg *alg;
 	int err;
 	if (!p)
 		return -ENOMEM;
 	alg = &p->alg;
 	snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
 	snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
 	alg->cra_priority = N2_CRA_PRIORITY;
 	alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
 	alg->cra_blocksize = tmpl->block_size;
 	p->enc_type = tmpl->enc_type;
 	alg->cra_ctxsize = sizeof(struct n2_cipher_context);
 	alg->cra_type = &crypto_ablkcipher_type;
 	alg->cra_u.ablkcipher = tmpl->ablkcipher;
 	alg->cra_init = n2_cipher_cra_init;
 	alg->cra_module = THIS_MODULE;
 	list_add(&p->entry, &cipher_algs);
 	err = crypto_register_alg(alg);
 	if (err) {
 		pr_err("%s alg registration failed\n", alg->cra_name);
 		list_del(&p->entry);
 		kfree(p);
 	} else {
 		pr_info("%s alg registered\n", alg->cra_name);
 	}
 	return err;
 }
 static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
 {
 	struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
 	struct ahash_alg *ahash;
 	struct crypto_alg *base;
 	int err;
 	if (!p)
 		return -ENOMEM;
 	p->child_alg = n2ahash->alg.halg.base.cra_name;
 	memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
 	INIT_LIST_HEAD(&p->derived.entry);
 	ahash = &p->derived.alg;
 	ahash->digest = n2_hmac_async_digest;
 	ahash->setkey = n2_hmac_async_setkey;
 	base = &ahash->halg.base;
 	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
 	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
 	base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
 	base->cra_init = n2_hmac_cra_init;
 	base->cra_exit = n2_hmac_cra_exit;
 	list_add(&p->derived.entry, &hmac_algs);
 	err = crypto_register_ahash(ahash);
 	if (err) {
 		pr_err("%s alg registration failed\n", base->cra_name);
 		list_del(&p->derived.entry);
 		kfree(p);
 	} else {
 		pr_info("%s alg registered\n", base->cra_name);
 	}
 	return err;
 }
 static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
 {
 	struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
 	struct hash_alg_common *halg;
 	struct crypto_alg *base;
 	struct ahash_alg *ahash;
 	int err;
 	if (!p)
 		return -ENOMEM;
 	p->hash_zero = tmpl->hash_zero;
 	p->hash_init = tmpl->hash_init;
 	p->auth_type = tmpl->auth_type;
 	p->hmac_type = tmpl->hmac_type;
 	p->hw_op_hashsz = tmpl->hw_op_hashsz;
 	p->digest_size = tmpl->digest_size;
 	ahash = &p->alg;
 	ahash->init = n2_hash_async_init;
 	ahash->update = n2_hash_async_update;
 	ahash->final = n2_hash_async_final;
 	ahash->finup = n2_hash_async_finup;
 	ahash->digest = n2_hash_async_digest;
 	halg = &ahash->halg;
 	halg->digestsize = tmpl->digest_size;
 	base = &halg->base;
 	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
 	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
 	base->cra_priority = N2_CRA_PRIORITY;
 	base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;
 	base->cra_blocksize = tmpl->block_size;
 	base->cra_ctxsize = sizeof(struct n2_hash_ctx);
 	base->cra_module = THIS_MODULE;
 	base->cra_init = n2_hash_cra_init;
 	base->cra_exit = n2_hash_cra_exit;
 	list_add(&p->entry, &ahash_algs);
 	err = crypto_register_ahash(ahash);
 	if (err) {
 		pr_err("%s alg registration failed\n", base->cra_name);
 		list_del(&p->entry);
 		kfree(p);
 	} else {
 		pr_info("%s alg registered\n", base->cra_name);
 	}
 	if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
 		err = __n2_register_one_hmac(p);
 	return err;
 }
 static int __devinit n2_register_algs(void)
 {
 	int i, err = 0;
 	mutex_lock(&spu_lock);
 	if (algs_registered++)
 		goto out;
 	for (i = 0; i < NUM_HASH_TMPLS; i++) {
 		err = __n2_register_one_ahash(&hash_tmpls[i]);
 		if (err) {
 			__n2_unregister_algs();
 			goto out;
 		}
 	}
 	for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
 		err = __n2_register_one_cipher(&cipher_tmpls[i]);
 		if (err) {
 			__n2_unregister_algs();
 			goto out;
 		}
 	}
 out:
 	mutex_unlock(&spu_lock);
 	return err;
 }
 static void __devexit n2_unregister_algs(void)
 {
 	mutex_lock(&spu_lock);
 	if (!--algs_registered)
 		__n2_unregister_algs();
 	mutex_unlock(&spu_lock);
 }
 /* To map CWQ queues to interrupt sources, the hypervisor API provides
  * a devino.  This isn't very useful to us because all of the
  * interrupts listed in the device_node have been translated to
  * Linux virtual IRQ cookie numbers.
  *
  * So we have to back-translate, going through the 'intr' and 'ino'
  * property tables of the n2cp MDESC node, matching it with the OF
  * 'interrupts' property entries, in order to to figure out which
  * devino goes to which already-translated IRQ.
  */
 static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
 			     unsigned long dev_ino)
 {
 	const unsigned int *dev_intrs;
 	unsigned int intr;
 	int i;
 	for (i = 0; i < ip->num_intrs; i++) {
 		if (ip->ino_table[i].ino == dev_ino)
 			break;
 	}
 	if (i == ip->num_intrs)
 		return -ENODEV;
 	intr = ip->ino_table[i].intr;
 	dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
 	if (!dev_intrs)
 		return -ENODEV;
 	for (i = 0; i < dev->archdata.num_irqs; i++) {
 		if (dev_intrs[i] == intr)
 			return i;
 	}
 	return -ENODEV;
 }
 static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
 		       const char *irq_name, struct spu_queue *p,
 		       irq_handler_t handler)
 {
 	unsigned long herr;
 	int index;
 	herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
 	if (herr)
 		return -EINVAL;
 	index = find_devino_index(dev, ip, p->devino);
 	if (index < 0)
 		return index;
 	p->irq = dev->archdata.irqs[index];
 	sprintf(p->irq_name, "%s-%d", irq_name, index);
 	return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,
 			   p->irq_name, p);
 }
 static struct kmem_cache *queue_cache[2];
 static void *new_queue(unsigned long q_type)
 {
 	return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
 }
 static void free_queue(void *p, unsigned long q_type)
 {
 	return kmem_cache_free(queue_cache[q_type - 1], p);
 }
 static int queue_cache_init(void)
 {
 	if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
 		queue_cache[HV_NCS_QTYPE_MAU - 1] =
 			kmem_cache_create("mau_queue",
 					  (MAU_NUM_ENTRIES *
 					   MAU_ENTRY_SIZE),
 					  MAU_ENTRY_SIZE, 0, NULL);
 	if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
 		return -ENOMEM;
 	if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
 		queue_cache[HV_NCS_QTYPE_CWQ - 1] =
 			kmem_cache_create("cwq_queue",
 					  (CWQ_NUM_ENTRIES *
 					   CWQ_ENTRY_SIZE),
 					  CWQ_ENTRY_SIZE, 0, NULL);
 	if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
 		kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
 		return -ENOMEM;
 	}
 	return 0;
 }
 static void queue_cache_destroy(void)
 {
 	kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
 	kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
 }
 static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
 {
 	cpumask_var_t old_allowed;
 	unsigned long hv_ret;
 	if (cpumask_empty(&p->sharing))
 		return -EINVAL;
 	if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
 		return -ENOMEM;
 	cpumask_copy(old_allowed, &current->cpus_allowed);
 	set_cpus_allowed_ptr(current, &p->sharing);
 	hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
 				 CWQ_NUM_ENTRIES, &p->qhandle);
 	if (!hv_ret)
 		sun4v_ncs_sethead_marker(p->qhandle, 0);
 	set_cpus_allowed_ptr(current, old_allowed);
 	free_cpumask_var(old_allowed);
 	return (hv_ret ? -EINVAL : 0);
 }
 static int spu_queue_setup(struct spu_queue *p)
 {
 	int err;
 	p->q = new_queue(p->q_type);
 	if (!p->q)
 		return -ENOMEM;
 	err = spu_queue_register(p, p->q_type);
 	if (err) {
 		free_queue(p->q, p->q_type);
 		p->q = NULL;
 	}
 	return err;
 }
 static void spu_queue_destroy(struct spu_queue *p)
 {
 	unsigned long hv_ret;
 	if (!p->q)
 		return;
 	hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
 	if (!hv_ret)
 		free_queue(p->q, p->q_type);
 }
 static void spu_list_destroy(struct list_head *list)
 {
 	struct spu_queue *p, *n;
 	list_for_each_entry_safe(p, n, list, list) {
 		int i;
 		for (i = 0; i < NR_CPUS; i++) {
 			if (cpu_to_cwq[i] == p)
 				cpu_to_cwq[i] = NULL;
 		}
 		if (p->irq) {
 			free_irq(p->irq, p);
 			p->irq = 0;
 		}
 		spu_queue_destroy(p);
 		list_del(&p->list);
 		kfree(p);
 	}
 }
 /* Walk the backward arcs of a CWQ 'exec-unit' node,
  * gathering cpu membership information.
  */
 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
 			       struct platform_device *dev,
 			       u64 node, struct spu_queue *p,
 			       struct spu_queue **table)
 {
 	u64 arc;
 	mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
 		u64 tgt = mdesc_arc_target(mdesc, arc);
 		const char *name = mdesc_node_name(mdesc, tgt);
 		const u64 *id;
 		if (strcmp(name, "cpu"))
 			continue;
 		id = mdesc_get_property(mdesc, tgt, "id", NULL);
 		if (table[*id] != NULL) {
 			dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
 				dev->dev.of_node->full_name);
 			return -EINVAL;
 		}
 		cpu_set(*id, p->sharing);
 		table[*id] = p;
 	}
 	return 0;
 }
 /* Process an 'exec-unit' MDESC node of type 'cwq'.  */
 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
 			    struct platform_device *dev, struct mdesc_handle *mdesc,
 			    u64 node, const char *iname, unsigned long q_type,
 			    irq_handler_t handler, struct spu_queue **table)
 {
 	struct spu_queue *p;
 	int err;
 	p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
 	if (!p) {
 		dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
 			dev->dev.of_node->full_name);
 		return -ENOMEM;
 	}
 	cpus_clear(p->sharing);
 	spin_lock_init(&p->lock);
 	p->q_type = q_type;
 	INIT_LIST_HEAD(&p->jobs);
 	list_add(&p->list, list);
 	err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
 	if (err)
 		return err;
 	err = spu_queue_setup(p);
 	if (err)
 		return err;
 	return spu_map_ino(dev, ip, iname, p, handler);
 }
 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
 			  struct spu_mdesc_info *ip, struct list_head *list,
 			  const char *exec_name, unsigned long q_type,
 			  irq_handler_t handler, struct spu_queue **table)
 {
 	int err = 0;
 	u64 node;
 	mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
 		const char *type;
 		type = mdesc_get_property(mdesc, node, "type", NULL);
 		if (!type || strcmp(type, exec_name))
 			continue;
 		err = handle_exec_unit(ip, list, dev, mdesc, node,
 				       exec_name, q_type, handler, table);
 		if (err) {
 			spu_list_destroy(list);
 			break;
 		}
 	}
 	return err;
 }
 static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
 				   struct spu_mdesc_info *ip)
 {
 	const u64 *ino;
 	int ino_len;
 	int i;
 	ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
 	if (!ino) {
 		printk("NO 'ino'\n");
 		return -ENODEV;
 	}
 	ip->num_intrs = ino_len / sizeof(u64);
 	ip->ino_table = kzalloc((sizeof(struct ino_blob) *
 				 ip->num_intrs),
 				GFP_KERNEL);
 	if (!ip->ino_table)
 		return -ENOMEM;
 	for (i = 0; i < ip->num_intrs; i++) {
 		struct ino_blob *b = &ip->ino_table[i];
 		b->intr = i + 1;
 		b->ino = ino[i];
 	}
 	return 0;
 }
 static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
 					  struct platform_device *dev,
 					  struct spu_mdesc_info *ip,
 					  const char *node_name)
 {
 	const unsigned int *reg;
 	u64 node;
 	reg = of_get_property(dev->dev.of_node, "reg", NULL);
 	if (!reg)
 		return -ENODEV;
 	mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
 		const char *name;
 		const u64 *chdl;
 		name = mdesc_get_property(mdesc, node, "name", NULL);
 		if (!name || strcmp(name, node_name))
 			continue;
 		chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
 		if (!chdl || (*chdl != *reg))
 			continue;
 		ip->cfg_handle = *chdl;
 		return get_irq_props(mdesc, node, ip);
 	}
 	return -ENODEV;
 }
 static unsigned long n2_spu_hvapi_major;
 static unsigned long n2_spu_hvapi_minor;
 static int __devinit n2_spu_hvapi_register(void)
 {
 	int err;
 	n2_spu_hvapi_major = 2;
 	n2_spu_hvapi_minor = 0;
 	err = sun4v_hvapi_register(HV_GRP_NCS,
 				   n2_spu_hvapi_major,
 				   &n2_spu_hvapi_minor);
 	if (!err)
 		pr_info("Registered NCS HVAPI version %lu.%lu\n",
 			n2_spu_hvapi_major,
 			n2_spu_hvapi_minor);
 	return err;
 }
 static void n2_spu_hvapi_unregister(void)
 {
 	sun4v_hvapi_unregister(HV_GRP_NCS);
 }
 static int global_ref;
 static int __devinit grab_global_resources(void)
 {
 	int err = 0;
 	mutex_lock(&spu_lock);
 	if (global_ref++)
 		goto out;
 	err = n2_spu_hvapi_register();
 	if (err)
 		goto out;
 	err = queue_cache_init();
 	if (err)
 		goto out_hvapi_release;
 	err = -ENOMEM;
 	cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
 			     GFP_KERNEL);
 	if (!cpu_to_cwq)
 		goto out_queue_cache_destroy;
 	cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
 			     GFP_KERNEL);
 	if (!cpu_to_mau)
 		goto out_free_cwq_table;
 	err = 0;
 out:
 	if (err)
 		global_ref--;
 	mutex_unlock(&spu_lock);
 	return err;
 out_free_cwq_table:
 	kfree(cpu_to_cwq);
 	cpu_to_cwq = NULL;
 out_queue_cache_destroy:
 	queue_cache_destroy();
 out_hvapi_release:
 	n2_spu_hvapi_unregister();
 	goto out;
 }
 static void release_global_resources(void)
 {
 	mutex_lock(&spu_lock);
 	if (!--global_ref) {
 		kfree(cpu_to_cwq);
 		cpu_to_cwq = NULL;
 		kfree(cpu_to_mau);
 		cpu_to_mau = NULL;
 		queue_cache_destroy();
 		n2_spu_hvapi_unregister();
 	}
 	mutex_unlock(&spu_lock);
 }
 static struct n2_crypto * __devinit alloc_n2cp(void)
 {
 	struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
 	if (np)
 		INIT_LIST_HEAD(&np->cwq_list);
 	return np;
 }
 static void free_n2cp(struct n2_crypto *np)
 {
 	if (np->cwq_info.ino_table) {
 		kfree(np->cwq_info.ino_table);
 		np->cwq_info.ino_table = NULL;
 	}
 	kfree(np);
 }
 static void __devinit n2_spu_driver_version(void)
 {
 	static int n2_spu_version_printed;
 	if (n2_spu_version_printed++ == 0)
 		pr_info("%s", version);
 }
 static int __devinit n2_crypto_probe(struct platform_device *dev)
 {
 	struct mdesc_handle *mdesc;
 	const char *full_name;
 	struct n2_crypto *np;
 	int err;
 	n2_spu_driver_version();
 	full_name = dev->dev.of_node->full_name;
 	pr_info("Found N2CP at %s\n", full_name);
 	np = alloc_n2cp();
 	if (!np) {
 		dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
 			full_name);
 		return -ENOMEM;
 	}
 	err = grab_global_resources();
 	if (err) {
 		dev_err(&dev->dev, "%s: Unable to grab "
 			"global resources.\n", full_name);
 		goto out_free_n2cp;
 	}
 	mdesc = mdesc_grab();
 	if (!mdesc) {
 		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
 			full_name);
 		err = -ENODEV;
 		goto out_free_global;
 	}
 	err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
 	if (err) {
 		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
 			full_name);
 		mdesc_release(mdesc);
 		goto out_free_global;
 	}
 	err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
 			     "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
 			     cpu_to_cwq);
 	mdesc_release(mdesc);
 	if (err) {
 		dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
 			full_name);
 		goto out_free_global;
 	}
 	err = n2_register_algs();
 	if (err) {
 		dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
 			full_name);
 		goto out_free_spu_list;
 	}
 	dev_set_drvdata(&dev->dev, np);
 	return 0;
 out_free_spu_list:
 	spu_list_destroy(&np->cwq_list);
 out_free_global:
 	release_global_resources();
 out_free_n2cp:
 	free_n2cp(np);
 	return err;
 }
 static int __devexit n2_crypto_remove(struct platform_device *dev)
 {
 	struct n2_crypto *np = dev_get_drvdata(&dev->dev);
 	n2_unregister_algs();
 	spu_list_destroy(&np->cwq_list);
 	release_global_resources();
 	free_n2cp(np);
 	return 0;
 }
 static struct n2_mau * __devinit alloc_ncp(void)
 {
 	struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
 	if (mp)
 		INIT_LIST_HEAD(&mp->mau_list);
 	return mp;
 }
 static void free_ncp(struct n2_mau *mp)
 {
 	if (mp->mau_info.ino_table) {
 		kfree(mp->mau_info.ino_table);
 		mp->mau_info.ino_table = NULL;
 	}
 	kfree(mp);
 }
 static int __devinit n2_mau_probe(struct platform_device *dev)
 {
 	struct mdesc_handle *mdesc;
 	const char *full_name;
 	struct n2_mau *mp;
 	int err;
 	n2_spu_driver_version();
 	full_name = dev->dev.of_node->full_name;
 	pr_info("Found NCP at %s\n", full_name);
 	mp = alloc_ncp();
 	if (!mp) {
 		dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
 			full_name);
 		return -ENOMEM;
 	}
 	err = grab_global_resources();
 	if (err) {
 		dev_err(&dev->dev, "%s: Unable to grab "
 			"global resources.\n", full_name);
 		goto out_free_ncp;
 	}
 	mdesc = mdesc_grab();
 	if (!mdesc) {
 		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
 			full_name);
 		err = -ENODEV;
 		goto out_free_global;
 	}
 	err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
 	if (err) {
 		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
 			full_name);
 		mdesc_release(mdesc);
 		goto out_free_global;
 	}
 	err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
 			     "mau", HV_NCS_QTYPE_MAU, mau_intr,
 			     cpu_to_mau);
 	mdesc_release(mdesc);
 	if (err) {
 		dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
 			full_name);
 		goto out_free_global;
 	}
 	dev_set_drvdata(&dev->dev, mp);
 	return 0;
 out_free_global:
 	release_global_resources();
 out_free_ncp:
 	free_ncp(mp);
 	return err;
 }
 static int __devexit n2_mau_remove(struct platform_device *dev)
 {
 	struct n2_mau *mp = dev_get_drvdata(&dev->dev);
 	spu_list_destroy(&mp->mau_list);
 	release_global_resources();
 	free_ncp(mp);
 	return 0;
 }
 static struct of_device_id n2_crypto_match[] = {
 	{
 		.name = "n2cp",
 		.compatible = "SUNW,n2-cwq",
 	},
 	{
 		.name = "n2cp",
 		.compatible = "SUNW,vf-cwq",
 	},
 	{
 		.name = "n2cp",
 		.compatible = "SUNW,kt-cwq",
 	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, n2_crypto_match);
 static struct platform_driver n2_crypto_driver = {
 	.driver = {
 		.name		=	"n2cp",
 		.owner		=	THIS_MODULE,
 		.of_match_table	=	n2_crypto_match,
 	},
 	.probe		=	n2_crypto_probe,
 	.remove		=	__devexit_p(n2_crypto_remove),
 };
 static struct of_device_id n2_mau_match[] = {
 	{
 		.name = "ncp",
 		.compatible = "SUNW,n2-mau",
 	},
 	{
 		.name = "ncp",
 		.compatible = "SUNW,vf-mau",
 	},
 	{
 		.name = "ncp",
 		.compatible = "SUNW,kt-mau",
 	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, n2_mau_match);
 static struct platform_driver n2_mau_driver = {
 	.driver = {
 		.name		=	"ncp",
 		.owner		=	THIS_MODULE,
 		.of_match_table	=	n2_mau_match,
 	},
 	.probe		=	n2_mau_probe,
 	.remove		=	__devexit_p(n2_mau_remove),
 };
 static int __init n2_init(void)
 {
 	int err = platform_driver_register(&n2_crypto_driver);
 	if (!err) {
 		err = platform_driver_register(&n2_mau_driver);
 		if (err)
 			platform_driver_unregister(&n2_crypto_driver);
 	}
 	return err;
 }
 static void __exit n2_exit(void)
 {
 	platform_driver_unregister(&n2_mau_driver);
 	platform_driver_unregister(&n2_crypto_driver);
 }
 module_init(n2_init);