checksum.S 8.91 KB
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		IP/TCP/UDP checksumming routines
 *
 * Xtensa version:  Copyright (C) 2001 Tensilica, Inc. by Kevin Chea
 *                  Optimized by Joe Taylor
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

#include <asm/errno.h>
#include <linux/linkage.h>
#include <variant/core.h>

/*
 * computes a partial checksum, e.g. for TCP/UDP fragments
 */

/*
 * unsigned int csum_partial(const unsigned char *buf, int len,
 *                           unsigned int sum);
 *    a2 = buf
 *    a3 = len
 *    a4 = sum
 *
 * This function assumes 2- or 4-byte alignment.  Other alignments will fail!
 */

/* ONES_ADD converts twos-complement math to ones-complement. */
#define ONES_ADD(sum, val)	  \
	add	sum, sum, val	; \
	bgeu	sum, val, 99f	; \
	addi	sum, sum, 1	; \
99:				;

.text
ENTRY(csum_partial)

	/*
	 * Experiments with Ethernet and SLIP connections show that buf
	 * is aligned on either a 2-byte or 4-byte boundary.
	 */
	entry	sp, 32
	extui	a5, a2, 0, 2
	bnez	a5, 8f		/* branch if 2-byte aligned */
	/* Fall-through on common case, 4-byte alignment */
1:
	srli	a5, a3, 5	/* 32-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a5, 2f
#else
	beqz	a5, 2f
	slli	a5, a5, 5
	add	a5, a5, a2	/* a5 = end of last 32-byte chunk */
.Loop1:
#endif
	l32i	a6, a2, 0
	l32i	a7, a2, 4
	ONES_ADD(a4, a6)
	ONES_ADD(a4, a7)
	l32i	a6, a2, 8
	l32i	a7, a2, 12
	ONES_ADD(a4, a6)
	ONES_ADD(a4, a7)
	l32i	a6, a2, 16
	l32i	a7, a2, 20
	ONES_ADD(a4, a6)
	ONES_ADD(a4, a7)
	l32i	a6, a2, 24
	l32i	a7, a2, 28
	ONES_ADD(a4, a6)
	ONES_ADD(a4, a7)
	addi	a2, a2, 4*8
#if !XCHAL_HAVE_LOOPS
	blt	a2, a5, .Loop1
#endif
2:
	extui	a5, a3, 2, 3	/* remaining 4-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a5, 3f
#else
	beqz	a5, 3f
	slli	a5, a5, 2
	add	a5, a5, a2	/* a5 = end of last 4-byte chunk */
.Loop2:
#endif
	l32i	a6, a2, 0
	ONES_ADD(a4, a6)
	addi	a2, a2, 4
#if !XCHAL_HAVE_LOOPS
	blt	a2, a5, .Loop2
#endif
3:
	_bbci.l	a3, 1, 5f	/* remaining 2-byte chunk */
	l16ui	a6, a2, 0
	ONES_ADD(a4, a6)
	addi	a2, a2, 2
5:
	_bbci.l	a3, 0, 7f	/* remaining 1-byte chunk */
6:	l8ui	a6, a2, 0
#ifdef __XTENSA_EB__
	slli	a6, a6, 8	/* load byte into bits 8..15 */
#endif
	ONES_ADD(a4, a6)
7:
	mov	a2, a4
	retw

	/* uncommon case, buf is 2-byte aligned */
8:
	beqz	a3, 7b		/* branch if len == 0 */
	beqi	a3, 1, 6b	/* branch if len == 1 */

	extui	a5, a2, 0, 1
	bnez	a5, 8f		/* branch if 1-byte aligned */

	l16ui	a6, a2, 0	/* common case, len >= 2 */
	ONES_ADD(a4, a6)
	addi	a2, a2, 2	/* adjust buf */
	addi	a3, a3, -2	/* adjust len */
	j	1b		/* now buf is 4-byte aligned */

	/* case: odd-byte aligned, len > 1
	 * This case is dog slow, so don't give us an odd address.
	 * (I don't think this ever happens, but just in case.)
	 */
8:
	srli	a5, a3, 2	/* 4-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a5, 2f
#else
	beqz	a5, 2f
	slli	a5, a5, 2
	add	a5, a5, a2	/* a5 = end of last 4-byte chunk */
.Loop3:
#endif
	l8ui	a6, a2, 0	/* bits 24..31 */
	l16ui	a7, a2, 1	/* bits  8..23 */
	l8ui	a8, a2, 3	/* bits  0.. 8 */
#ifdef	__XTENSA_EB__
	slli	a6, a6, 24
#else
	slli	a8, a8, 24
#endif
	slli	a7, a7, 8
	or	a7, a7, a6
	or	a7, a7, a8
	ONES_ADD(a4, a7)
	addi	a2, a2, 4
#if !XCHAL_HAVE_LOOPS
	blt	a2, a5, .Loop3
#endif
2:
	_bbci.l	a3, 1, 3f	/* remaining 2-byte chunk, still odd addr */
	l8ui	a6, a2, 0
	l8ui	a7, a2, 1
#ifdef	__XTENSA_EB__
	slli	a6, a6, 8
#else
	slli	a7, a7, 8
#endif
	or	a7, a7, a6
	ONES_ADD(a4, a7)
	addi	a2, a2, 2
3:
	j	5b		/* branch to handle the remaining byte */

ENDPROC(csum_partial)

/*
 * Copy from ds while checksumming, otherwise like csum_partial
 *
 * The macros SRC and DST specify the type of access for the instruction.
 * thus we can call a custom exception handler for each access type.
 */

#define SRC(y...)			\
	9999: y;			\
	.section __ex_table, "a";	\
	.long 9999b, 6001f	;	\
	.previous

#define DST(y...)			\
	9999: y;			\
	.section __ex_table, "a";	\
	.long 9999b, 6002f	;	\
	.previous

/*
unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,
					int sum, int *src_err_ptr, int *dst_err_ptr)
	a2  = src
	a3  = dst
	a4  = len
	a5  = sum
	a6  = src_err_ptr
	a7  = dst_err_ptr
	a8  = temp
	a9  = temp
	a10 = temp
	a11 = original len for exception handling
	a12 = original dst for exception handling

    This function is optimized for 4-byte aligned addresses.  Other
    alignments work, but not nearly as efficiently.
 */

ENTRY(csum_partial_copy_generic)

	entry	sp, 32
	mov	a12, a3
	mov	a11, a4
	or	a10, a2, a3

	/* We optimize the following alignment tests for the 4-byte
	aligned case.  Two bbsi.l instructions might seem more optimal
	(commented out below).  However, both labels 5: and 3: are out
	of the imm8 range, so the assembler relaxes them into
	equivalent bbci.l, j combinations, which is actually
	slower. */

	extui	a9, a10, 0, 2
	beqz	a9, 1f		/* branch if both are 4-byte aligned */
	bbsi.l	a10, 0, 5f	/* branch if one address is odd */
	j	3f		/* one address is 2-byte aligned */

/*	_bbsi.l	a10, 0, 5f */	/* branch if odd address */
/*	_bbsi.l	a10, 1, 3f */	/* branch if 2-byte-aligned address */

1:
	/* src and dst are both 4-byte aligned */
	srli	a10, a4, 5	/* 32-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a10, 2f
#else
	beqz	a10, 2f
	slli	a10, a10, 5
	add	a10, a10, a2	/* a10 = end of last 32-byte src chunk */
.Loop5:
#endif
SRC(	l32i	a9, a2, 0	)
SRC(	l32i	a8, a2, 4	)
DST(	s32i	a9, a3, 0	)
DST(	s32i	a8, a3, 4	)
	ONES_ADD(a5, a9)
	ONES_ADD(a5, a8)
SRC(	l32i	a9, a2, 8	)
SRC(	l32i	a8, a2, 12	)
DST(	s32i	a9, a3, 8	)
DST(	s32i	a8, a3, 12	)
	ONES_ADD(a5, a9)
	ONES_ADD(a5, a8)
SRC(	l32i	a9, a2, 16	)
SRC(	l32i	a8, a2, 20	)
DST(	s32i	a9, a3, 16	)
DST(	s32i	a8, a3, 20	)
	ONES_ADD(a5, a9)
	ONES_ADD(a5, a8)
SRC(	l32i	a9, a2, 24	)
SRC(	l32i	a8, a2, 28	)
DST(	s32i	a9, a3, 24	)
DST(	s32i	a8, a3, 28	)
	ONES_ADD(a5, a9)
	ONES_ADD(a5, a8)
	addi	a2, a2, 32
	addi	a3, a3, 32
#if !XCHAL_HAVE_LOOPS
	blt	a2, a10, .Loop5
#endif
2:
	extui	a10, a4, 2, 3	/* remaining 4-byte chunks */
	extui	a4, a4, 0, 2	/* reset len for general-case, 2-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a10, 3f
#else
	beqz	a10, 3f
	slli	a10, a10, 2
	add	a10, a10, a2	/* a10 = end of last 4-byte src chunk */
.Loop6:
#endif
SRC(	l32i	a9, a2, 0	)
DST(	s32i	a9, a3, 0	)
	ONES_ADD(a5, a9)
	addi	a2, a2, 4
	addi	a3, a3, 4
#if !XCHAL_HAVE_LOOPS
	blt	a2, a10, .Loop6
#endif
3:
	/*
	Control comes to here in two cases: (1) It may fall through
	to here from the 4-byte alignment case to process, at most,
	one 2-byte chunk.  (2) It branches to here from above if
	either src or dst is 2-byte aligned, and we process all bytes
	here, except for perhaps a trailing odd byte.  It's
	inefficient, so align your addresses to 4-byte boundaries.

	a2 = src
	a3 = dst
	a4 = len
	a5 = sum
	*/
	srli	a10, a4, 1	/* 2-byte chunks */
#if XCHAL_HAVE_LOOPS
	loopgtz	a10, 4f
#else
	beqz	a10, 4f
	slli	a10, a10, 1
	add	a10, a10, a2	/* a10 = end of last 2-byte src chunk */
.Loop7:
#endif
SRC(	l16ui	a9, a2, 0	)
DST(	s16i	a9, a3, 0	)
	ONES_ADD(a5, a9)
	addi	a2, a2, 2
	addi	a3, a3, 2
#if !XCHAL_HAVE_LOOPS
	blt	a2, a10, .Loop7
#endif
4:
	/* This section processes a possible trailing odd byte. */
	_bbci.l	a4, 0, 8f	/* 1-byte chunk */
SRC(	l8ui	a9, a2, 0	)
DST(	s8i	a9, a3, 0	)
#ifdef __XTENSA_EB__
	slli	a9, a9, 8	/* shift byte to bits 8..15 */
#endif
	ONES_ADD(a5, a9)
8:
	mov	a2, a5
	retw

5:
	/* Control branch to here when either src or dst is odd.  We
	process all bytes using 8-bit accesses.  Grossly inefficient,
	so don't feed us an odd address. */

	srli	a10, a4, 1	/* handle in pairs for 16-bit csum */
#if XCHAL_HAVE_LOOPS
	loopgtz	a10, 6f
#else
	beqz	a10, 6f
	slli	a10, a10, 1
	add	a10, a10, a2	/* a10 = end of last odd-aligned, 2-byte src chunk */
.Loop8:
#endif
SRC(	l8ui	a9, a2, 0	)
SRC(	l8ui	a8, a2, 1	)
DST(	s8i	a9, a3, 0	)
DST(	s8i	a8, a3, 1	)
#ifdef __XTENSA_EB__
	slli	a9, a9, 8	/* combine into a single 16-bit value */
#else				/* for checksum computation */
	slli	a8, a8, 8
#endif
	or	a9, a9, a8
	ONES_ADD(a5, a9)
	addi	a2, a2, 2
	addi	a3, a3, 2
#if !XCHAL_HAVE_LOOPS
	blt	a2, a10, .Loop8
#endif
6:
	j	4b		/* process the possible trailing odd byte */

ENDPROC(csum_partial_copy_generic)


# Exception handler:
.section .fixup, "ax"
/*
	a6  = src_err_ptr
	a7  = dst_err_ptr
	a11 = original len for exception handling
	a12 = original dst for exception handling
*/

6001:
	_movi	a2, -EFAULT
	s32i	a2, a6, 0	/* src_err_ptr */

	# clear the complete destination - computing the rest
	# is too much work
	movi	a2, 0
#if XCHAL_HAVE_LOOPS
	loopgtz	a11, 2f
#else
	beqz	a11, 2f
	add	a11, a11, a12	/* a11 = ending address */
.Leloop:
#endif
	s8i	a2, a12, 0
	addi	a12, a12, 1
#if !XCHAL_HAVE_LOOPS
	blt	a12, a11, .Leloop
#endif
2:
	retw

6002:
	movi	a2, -EFAULT
	s32i	a2, a7, 0	/* dst_err_ptr */
	movi	a2, 0
	retw

.previous