/* $Id: VIScopy.S,v 1.20 1999/05/25 16:52:57 jj Exp $
* VIScopy.S: High speed copy operations utilizing the UltraSparc
* Visual Instruction Set.
*
* Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
* Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
*/
#include "VIS.h"
/* VIS code can be used for numerous copy/set operation variants.
* It can be made to work in the kernel, one single instance,
* for all of memcpy, copy_to_user, and copy_from_user by setting
* the ASI src/dest globals correctly. Furthermore it can
* be used for kernel-->kernel page copies as well, a hook label
* is put in here just for this purpose.
*
* For userland, compiling this without __KERNEL__ defined makes
* it work just fine as a generic libc bcopy and memcpy.
* If for userland it is compiled with a 32bit gcc (but you need
* -Wa,-Av9a for as), the code will just rely on lower 32bits of
* IEU registers, if you compile it with 64bit gcc (ie. define
* __sparc_v9__), the code will use full 64bit.
*/
#ifdef __KERNEL__
#include <asm/visasm.h>
#include <asm/asm_offsets.h>
#define FPU_CLEAN_RETL \
ldub [%g6 + AOFF_task_tss + AOFF_thread_current_ds], %o1; \
VISExit \
clr %o0; \
retl; \
wr %o1, %g0, %asi;
#define FPU_RETL \
ldub [%g6 + AOFF_task_tss + AOFF_thread_current_ds], %o1; \
VISExit \
clr %o0; \
retl; \
wr %o1, %g0, %asi;
#define NORMAL_RETL \
ldub [%g6 + AOFF_task_tss + AOFF_thread_current_ds], %o1; \
clr %o0; \
retl; \
wr %o1, %g0, %asi;
#define EX(x,y,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: ba VIScopyfixup_ret; \
a, b, %o1; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EX2(x,y,c,d,e,a,b) \
98: x,y; \
.section .fixup; \
.align 4; \
99: c, d, e; \
ba VIScopyfixup_ret; \
a, b, %o1; \
.section __ex_table; \
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
#define EXO2(x,y) \
98: x,y; \
.section __ex_table; \
.align 4; \
.word 98b, VIScopyfixup_reto2; \
.text; \
.align 4;
#define EXVISN(x,y,n) \
98: x,y; \
.section __ex_table; \
.align 4; \
.word 98b, VIScopyfixup_vis##n; \
.text; \
.align 4;
#define EXT(start,end,handler) \
.section __ex_table; \
.align 4; \
.word start, 0, end, handler; \
.text; \
.align 4;
#else
#ifdef REGS_64BIT
#define FPU_CLEAN_RETL \
retl; \
mov %g6, %o0;
#define FPU_RETL \
retl; \
mov %g6, %o0;
#else
#define FPU_CLEAN_RETL \
wr %g0, FPRS_FEF, %fprs; \
retl; \
mov %g6, %o0;
#define FPU_RETL \
wr %g0, FPRS_FEF, %fprs; \
retl; \
mov %g6, %o0;
#endif
#define NORMAL_RETL \
retl; \
mov %g6, %o0;
#define EX(x,y,a,b) x,y
#define EX2(x,y,c,d,e,a,b) x,y
#define EXO2(x,y) x,y
#define EXVISN(x,y,n) x,y
#define EXT(a,b,c)
#endif
#define EXVIS(x,y) EXVISN(x,y,0)
#define EXVIS1(x,y) EXVISN(x,y,1)
#define EXVIS2(x,y) EXVISN(x,y,2)
#define EXVIS3(x,y) EXVISN(x,y,3)
#define EXVIS4(x,y) EXVISN(x,y,4)
#define EXVIS5(x,y) EXVISN(x,y,5)
#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
faligndata %f1, %f2, %f48; \
faligndata %f2, %f3, %f50; \
faligndata %f3, %f4, %f52; \
faligndata %f4, %f5, %f54; \
faligndata %f5, %f6, %f56; \
faligndata %f6, %f7, %f58; \
faligndata %f7, %f8, %f60; \
faligndata %f8, %f9, %f62;
#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
EXVIS(LDBLK [%src] ASIBLK, %fdest); \
ASI_SETDST_BLK \
add %src, 0x40, %src; \
add %dest, 0x40, %dest; \
subcc %len, 0x40, %len; \
be,pn %xcc, jmptgt; \
EXVIS2(STBLK %fsrc, [%dest - 0x40] ASIBLK); \
ASI_SETSRC_BLK
#define LOOP_CHUNK1(src, dest, len, branch_dest) \
MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
#define LOOP_CHUNK2(src, dest, len, branch_dest) \
MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
#define LOOP_CHUNK3(src, dest, len, branch_dest) \
MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
#define STORE_SYNC(dest, fsrc) \
EXVIS(STBLK %fsrc, [%dest] ASIBLK); \
add %dest, 0x40, %dest;
#ifdef __KERNEL__
#define STORE_JUMP(dest, fsrc, target) \
srl asi_dest, 3, %g5; \
EXVIS3(STBLK %fsrc, [%dest] ASIBLK); \
xor asi_dest, ASI_BLK_XOR1, asi_dest;\
add %dest, 0x40, %dest; \
xor asi_dest, %g5, asi_dest; \
ba,pt %xcc, target;
#else
#define STORE_JUMP(dest, fsrc, target) \
EXVIS3(STBLK %fsrc, [%dest] ASIBLK); \
add %dest, 0x40, %dest; \
ba,pt %xcc, target;
#endif
#ifndef __KERNEL__
#define VISLOOP_PAD nop; nop; nop; nop; \
nop; nop; nop; nop; \
nop; nop; nop; nop; \
nop; nop; nop;
#else
#define VISLOOP_PAD
#endif
#define FINISH_VISCHUNK(dest, f0, f1, left) \
ASI_SETDST_NOBLK \
subcc %left, 8, %left; \
bl,pn %xcc, vis_out; \
faligndata %f0, %f1, %f48; \
EXVIS4(STDF %f48, [%dest] ASINORMAL); \
add %dest, 8, %dest;
#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
subcc %left, 8, %left; \
bl,pn %xcc, vis_out; \
fsrc1 %f0, %f1;
#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
ba,a,pt %xcc, vis_out_slk;
/* Macros for non-VIS memcpy code. */
#ifdef REGS_64BIT
#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
ASI_SETSRC_NOBLK \
LDX [%src + offset + 0x00] ASINORMAL, %t0; \
LDX [%src + offset + 0x08] ASINORMAL, %t1; \
LDX [%src + offset + 0x10] ASINORMAL, %t2; \
LDX [%src + offset + 0x18] ASINORMAL, %t3; \
ASI_SETDST_NOBLK \
STW %t0, [%dst + offset + 0x04] ASINORMAL; \
srlx %t0, 32, %t0; \
STW %t0, [%dst + offset + 0x00] ASINORMAL; \
STW %t1, [%dst + offset + 0x0c] ASINORMAL; \
srlx %t1, 32, %t1; \
STW %t1, [%dst + offset + 0x08] ASINORMAL; \
STW %t2, [%dst + offset + 0x14] ASINORMAL; \
srlx %t2, 32, %t2; \
STW %t2, [%dst + offset + 0x10] ASINORMAL; \
STW %t3, [%dst + offset + 0x1c] ASINORMAL; \
srlx %t3, 32, %t3; \
STW %t3, [%dst + offset + 0x18] ASINORMAL;
#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
ASI_SETSRC_NOBLK \
LDX [%src + offset + 0x00] ASINORMAL, %t0; \
LDX [%src + offset + 0x08] ASINORMAL, %t1; \
LDX [%src + offset + 0x10] ASINORMAL, %t2; \
LDX [%src + offset + 0x18] ASINORMAL, %t3; \
ASI_SETDST_NOBLK \
STX %t0, [%dst + offset + 0x00] ASINORMAL; \
STX %t1, [%dst + offset + 0x08] ASINORMAL; \
STX %t2, [%dst + offset + 0x10] ASINORMAL; \
STX %t3, [%dst + offset + 0x18] ASINORMAL; \
ASI_SETSRC_NOBLK \
LDX [%src + offset + 0x20] ASINORMAL, %t0; \
LDX [%src + offset + 0x28] ASINORMAL, %t1; \
LDX [%src + offset + 0x30] ASINORMAL, %t2; \
LDX [%src + offset + 0x38] ASINORMAL, %t3; \
ASI_SETDST_NOBLK \
STX %t0, [%dst + offset + 0x20] ASINORMAL; \
STX %t1, [%dst + offset + 0x28] ASINORMAL; \
STX %t2, [%dst + offset + 0x30] ASINORMAL; \
STX %t3, [%dst + offset + 0x38] ASINORMAL;
#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
ASI_SETSRC_NOBLK \
LDX [%src - offset - 0x10] ASINORMAL, %t0; \
LDX [%src - offset - 0x08] ASINORMAL, %t1; \
ASI_SETDST_NOBLK \
STW %t0, [%dst - offset - 0x0c] ASINORMAL; \
srlx %t0, 32, %t2; \
STW %t2, [%dst - offset - 0x10] ASINORMAL; \
STW %t1, [%dst - offset - 0x04] ASINORMAL; \
srlx %t1, 32, %t3; \
STW %t3, [%dst - offset - 0x08] ASINORMAL;
#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1) \
ASI_SETSRC_NOBLK \
LDX [%src - offset - 0x10] ASINORMAL, %t0; \
LDX [%src - offset - 0x08] ASINORMAL, %t1; \
ASI_SETDST_NOBLK \
STX %t0, [%dst - offset - 0x10] ASINORMAL; \
STX %t1, [%dst - offset - 0x08] ASINORMAL;
#else /* !REGS_64BIT */
#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
lduw [%src + offset + 0x00], %t0; \
lduw [%src + offset + 0x04], %t1; \
lduw [%src + offset + 0x08], %t2; \
lduw [%src + offset + 0x0c], %t3; \
stw %t0, [%dst + offset + 0x00]; \
stw %t1, [%dst + offset + 0x04]; \
stw %t2, [%dst + offset + 0x08]; \
stw %t3, [%dst + offset + 0x0c]; \
lduw [%src + offset + 0x10], %t0; \
lduw [%src + offset + 0x14], %t1; \
lduw [%src + offset + 0x18], %t2; \
lduw [%src + offset + 0x1c], %t3; \
stw %t0, [%dst + offset + 0x10]; \
stw %t1, [%dst + offset + 0x14]; \
stw %t2, [%dst + offset + 0x18]; \
stw %t3, [%dst + offset + 0x1c];
#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
lduw [%src - offset - 0x10], %t0; \
lduw [%src - offset - 0x0c], %t1; \
lduw [%src - offset - 0x08], %t2; \
lduw [%src - offset - 0x04], %t3; \
stw %t0, [%dst - offset - 0x10]; \
stw %t1, [%dst - offset - 0x0c]; \
stw %t2, [%dst - offset - 0x08]; \
stw %t3, [%dst - offset - 0x04];
#endif /* !REGS_64BIT */
#ifdef __KERNEL__
.section __ex_table,#alloc
.section .fixup,#alloc,#execinstr
#endif
.text
.align 32
.globl memcpy
.type memcpy,@function
.globl bcopy
.type bcopy,@function
#ifdef __KERNEL__
.globl __memcpy
.type __memcpy,@function
.globl __memcpy_384plus
.type __memcpy_384plus,@function
.globl __memcpy_16plus
.type __memcpy_16plus,@function
.globl __memcpy_short
.type __memcpy_short,@function
.globl __memcpy_entry
.type __memcpy_entry,@function
memcpy_private:
__memcpy:
memcpy: mov ASI_P, asi_src ! IEU0 Group
brnz,pt %o2, __memcpy_entry ! CTI
mov ASI_P, asi_dest ! IEU1
retl
clr %o0
.align 32
.globl __copy_from_user
.type __copy_from_user,@function
__copy_from_user:rd %asi, asi_src ! IEU0 Group
brnz,pt %o2, __memcpy_entry ! CTI
mov ASI_P, asi_dest ! IEU1
.globl __copy_to_user
.type __copy_to_user,@function
__copy_to_user: mov ASI_P, asi_src ! IEU0 Group
brnz,pt %o2, __memcpy_entry ! CTI
rd %asi, asi_dest ! IEU1
retl ! CTI Group
clr %o0 ! IEU0 Group
.globl __copy_in_user
.type __copy_in_user,@function
__copy_in_user: rd %asi, asi_src ! IEU0 Group
brnz,pt %o2, __memcpy_entry ! CTI
mov asi_src, asi_dest ! IEU1
retl ! CTI Group
clr %o0 ! IEU0 Group
#endif
bcopy: or %o0, 0, %g3 ! IEU0 Group
addcc %o1, 0, %o0 ! IEU1
brgez,pt %o2, memcpy_private ! CTI
or %g3, 0, %o1 ! IEU0 Group
retl ! CTI Group brk forced
clr %o0 ! IEU0
.align 32
#ifdef __KERNEL__
__memcpy_384plus:
andcc %o0, 7, %g2 ! IEU1 Group
#endif
VIS_enter:
be,pt %xcc, dest_is_8byte_aligned ! CTI
#ifdef __KERNEL__
nop ! IEU0 Group
#else
andcc %o0, 0x38, %g5 ! IEU1 Group
#endif
do_dest_8byte_align:
mov 8, %g1 ! IEU0
sub %g1, %g2, %g2 ! IEU0 Group
andcc %o0, 1, %g0 ! IEU1
be,pt %icc, 2f ! CTI
sub %o2, %g2, %o2 ! IEU0 Group
1: ASI_SETSRC_NOBLK ! LSU Group
EX(LDUB [%o1] ASINORMAL, %o5,
add %o2, %g2) ! Load Group
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
subcc %g2, 1, %g2 ! IEU1 Group
be,pn %xcc, 3f ! CTI
EX2(STB %o5, [%o0 - 1] ASINORMAL,
add %g2, 1, %g2,
add %o2, %g2) ! Store
2: ASI_SETSRC_NOBLK ! LSU Group
EX(LDUB [%o1] ASINORMAL, %o5,
add %o2, %g2) ! Load Group
add %o0, 2, %o0 ! IEU0
EX2(LDUB [%o1 + 1] ASINORMAL, %g3,
sub %o0, 2, %o0,
add %o2, %g2) ! Load Group
ASI_SETDST_NOBLK ! LSU Group
subcc %g2, 2, %g2 ! IEU1 Group
EX2(STB %o5, [%o0 - 2] ASINORMAL,
add %g2, 2, %g2,
add %o2, %g2) ! Store
add %o1, 2, %o1 ! IEU0
bne,pt %xcc, 2b ! CTI Group
EX2(STB %g3, [%o0 - 1] ASINORMAL,
add %g2, 1, %g2,
add %o2, %g2) ! Store
#ifdef __KERNEL__
3:
dest_is_8byte_aligned:
VISEntry
andcc %o0, 0x38, %g5 ! IEU1 Group
#else
3: andcc %o0, 0x38, %g5 ! IEU1 Group
dest_is_8byte_aligned:
#endif
be,pt %icc, dest_is_64byte_aligned ! CTI
mov 64, %g1 ! IEU0
fmovd %f0, %f2 ! FPU
sub %g1, %g5, %g5 ! IEU0 Group
ASI_SETSRC_NOBLK ! LSU Group
alignaddr %o1, %g0, %g1 ! GRU Group
EXO2(LDDF [%g1] ASINORMAL, %f4) ! Load Group
sub %o2, %g5, %o2 ! IEU0
1: EX(LDDF [%g1 + 0x8] ASINORMAL, %f6,
add %o2, %g5) ! Load Group
add %g1, 0x8, %g1 ! IEU0 Group
subcc %g5, 8, %g5 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f4, %f6, %f0 ! GRU Group
EX2(STDF %f0, [%o0] ASINORMAL,
add %g5, 8, %g5,
add %o2, %g5) ! Store
add %o1, 8, %o1 ! IEU0 Group
be,pn %xcc, dest_is_64byte_aligned ! CTI
add %o0, 8, %o0 ! IEU1
ASI_SETSRC_NOBLK ! LSU Group
EX(LDDF [%g1 + 0x8] ASINORMAL, %f4,
add %o2, %g5) ! Load Group
add %g1, 8, %g1 ! IEU0
subcc %g5, 8, %g5 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f6, %f4, %f0 ! GRU Group
EX2(STDF %f0, [%o0] ASINORMAL,
add %g5, 8, %g5,
add %o2, %g5) ! Store
add %o1, 8, %o1 ! IEU0
ASI_SETSRC_NOBLK ! LSU Group
bne,pt %xcc, 1b ! CTI Group
add %o0, 8, %o0 ! IEU0
dest_is_64byte_aligned:
membar #LoadStore | #StoreStore | #StoreLoad ! LSU Group
#ifndef __KERNEL__
wr %g0, ASI_BLK_P, %asi ! LSU Group
#endif
subcc %o2, 0x40, %g7 ! IEU1 Group
mov %o1, %g1 ! IEU0
andncc %g7, (0x40 - 1), %g7 ! IEU1 Group
srl %g1, 3, %g2 ! IEU0
sub %o2, %g7, %g3 ! IEU0 Group
andn %o1, (0x40 - 1), %o1 ! IEU1
and %g2, 7, %g2 ! IEU0 Group
andncc %g3, 0x7, %g3 ! IEU1
fmovd %f0, %f2 ! FPU
sub %g3, 0x10, %g3 ! IEU0 Group
sub %o2, %g7, %o2 ! IEU1
#ifdef __KERNEL__
or asi_src, ASI_BLK_OR, asi_src ! IEU0 Group
or asi_dest, ASI_BLK_OR, asi_dest ! IEU1
#endif
alignaddr %g1, %g0, %g0 ! GRU Group
add %g1, %g7, %g1 ! IEU0 Group
subcc %o2, %g3, %o2 ! IEU1
ASI_SETSRC_BLK ! LSU Group
EXVIS1(LDBLK [%o1 + 0x00] ASIBLK, %f0) ! LSU Group
add %g1, %g3, %g1 ! IEU0
EXVIS1(LDBLK [%o1 + 0x40] ASIBLK, %f16) ! LSU Group
sub %g7, 0x80, %g7 ! IEU0
EXVIS(LDBLK [%o1 + 0x80] ASIBLK, %f32) ! LSU Group
#ifdef __KERNEL__
vispc: sll %g2, 9, %g2 ! IEU0 Group
sethi %hi(vis00), %g5 ! IEU1
or %g5, %lo(vis00), %g5 ! IEU0 Group
jmpl %g5 + %g2, %g0 ! CTI Group brk forced
addcc %o1, 0xc0, %o1 ! IEU1 Group
#else
! Clk1 Group 8-(
! Clk2 Group 8-(
! Clk3 Group 8-(
! Clk4 Group 8-(
vispc: rd %pc, %g5 ! PDU Group 8-(
addcc %g5, %lo(vis00 - vispc), %g5 ! IEU1 Group
sll %g2, 9, %g2 ! IEU0
jmpl %g5 + %g2, %g0 ! CTI Group brk forced
addcc %o1, 0xc0, %o1 ! IEU1 Group
#endif
.align 512 /* OK, here comes the fun part... */
vis00:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) LOOP_CHUNK1(o1, o0, g7, vis01)
FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) LOOP_CHUNK2(o1, o0, g7, vis02)
FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) LOOP_CHUNK3(o1, o0, g7, vis03)
b,pt %xcc, vis00+4; faligndata %f0, %f2, %f48
vis01:FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) STORE_JUMP(o0, f48, finish_f0) membar #Sync
vis02:FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_JUMP(o0, f48, finish_f16) membar #Sync
vis03:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_JUMP(o0, f48, finish_f32) membar #Sync
VISLOOP_PAD
vis10:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) LOOP_CHUNK1(o1, o0, g7, vis11)
FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) LOOP_CHUNK2(o1, o0, g7, vis12)
FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) LOOP_CHUNK3(o1, o0, g7, vis13)
b,pt %xcc, vis10+4; faligndata %f2, %f4, %f48
vis11:FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) STORE_JUMP(o0, f48, finish_f2) membar #Sync
vis12:FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_JUMP(o0, f48, finish_f18) membar #Sync
vis13:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_JUMP(o0, f48, finish_f34) membar #Sync
VISLOOP_PAD
vis20:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) LOOP_CHUNK1(o1, o0, g7, vis21)
FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) LOOP_CHUNK2(o1, o0, g7, vis22)
FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) LOOP_CHUNK3(o1, o0, g7, vis23)
b,pt %xcc, vis20+4; faligndata %f4, %f6, %f48
vis21:FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) STORE_JUMP(o0, f48, finish_f4) membar #Sync
vis22:FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_JUMP(o0, f48, finish_f20) membar #Sync
vis23:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_JUMP(o0, f48, finish_f36) membar #Sync
VISLOOP_PAD
vis30:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) LOOP_CHUNK1(o1, o0, g7, vis31)
FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) LOOP_CHUNK2(o1, o0, g7, vis32)
FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) LOOP_CHUNK3(o1, o0, g7, vis33)
b,pt %xcc, vis30+4; faligndata %f6, %f8, %f48
vis31:FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) STORE_JUMP(o0, f48, finish_f6) membar #Sync
vis32:FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_JUMP(o0, f48, finish_f22) membar #Sync
vis33:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_JUMP(o0, f48, finish_f38) membar #Sync
VISLOOP_PAD
vis40:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) LOOP_CHUNK1(o1, o0, g7, vis41)
FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) LOOP_CHUNK2(o1, o0, g7, vis42)
FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) LOOP_CHUNK3(o1, o0, g7, vis43)
b,pt %xcc, vis40+4; faligndata %f8, %f10, %f48
vis41:FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) STORE_JUMP(o0, f48, finish_f8) membar #Sync
vis42:FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_JUMP(o0, f48, finish_f24) membar #Sync
vis43:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_JUMP(o0, f48, finish_f40) membar #Sync
VISLOOP_PAD
vis50:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) LOOP_CHUNK1(o1, o0, g7, vis51)
FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) LOOP_CHUNK2(o1, o0, g7, vis52)
FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) LOOP_CHUNK3(o1, o0, g7, vis53)
b,pt %xcc, vis50+4; faligndata %f10, %f12, %f48
vis51:FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_JUMP(o0, f48, finish_f10) membar #Sync
vis52:FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_JUMP(o0, f48, finish_f26) membar #Sync
vis53:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_JUMP(o0, f48, finish_f42) membar #Sync
VISLOOP_PAD
vis60:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) LOOP_CHUNK1(o1, o0, g7, vis61)
FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) LOOP_CHUNK2(o1, o0, g7, vis62)
FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) LOOP_CHUNK3(o1, o0, g7, vis63)
b,pt %xcc, vis60+4; faligndata %f12, %f14, %f48
vis61:FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_JUMP(o0, f48, finish_f12) membar #Sync
vis62:FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_JUMP(o0, f48, finish_f28) membar #Sync
vis63:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_JUMP(o0, f48, finish_f44) membar #Sync
VISLOOP_PAD
vis70:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) LOOP_CHUNK1(o1, o0, g7, vis71)
FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) LOOP_CHUNK2(o1, o0, g7, vis72)
FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) LOOP_CHUNK3(o1, o0, g7, vis73)
b,pt %xcc, vis70+4; faligndata %f14, %f16, %f48
vis71:FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_JUMP(o0, f48, finish_f14) membar #Sync
vis72:FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_JUMP(o0, f48, finish_f30) membar #Sync
vis73:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_SYNC(o0, f48) membar #Sync
FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, finish_f46) membar #Sync
VISLOOP_PAD
finish_f0: FINISH_VISCHUNK(o0, f0, f2, g3)
finish_f2: FINISH_VISCHUNK(o0, f2, f4, g3)
finish_f4: FINISH_VISCHUNK(o0, f4, f6, g3)
finish_f6: FINISH_VISCHUNK(o0, f6, f8, g3)
finish_f8: FINISH_VISCHUNK(o0, f8, f10, g3)
finish_f10: FINISH_VISCHUNK(o0, f10, f12, g3)
finish_f12: FINISH_VISCHUNK(o0, f12, f14, g3)
finish_f14: UNEVEN_VISCHUNK(o0, f14, f0, g3)
finish_f16: FINISH_VISCHUNK(o0, f16, f18, g3)
finish_f18: FINISH_VISCHUNK(o0, f18, f20, g3)
finish_f20: FINISH_VISCHUNK(o0, f20, f22, g3)
finish_f22: FINISH_VISCHUNK(o0, f22, f24, g3)
finish_f24: FINISH_VISCHUNK(o0, f24, f26, g3)
finish_f26: FINISH_VISCHUNK(o0, f26, f28, g3)
finish_f28: FINISH_VISCHUNK(o0, f28, f30, g3)
finish_f30: UNEVEN_VISCHUNK(o0, f30, f0, g3)
finish_f32: FINISH_VISCHUNK(o0, f32, f34, g3)
finish_f34: FINISH_VISCHUNK(o0, f34, f36, g3)
finish_f36: FINISH_VISCHUNK(o0, f36, f38, g3)
finish_f38: FINISH_VISCHUNK(o0, f38, f40, g3)
finish_f40: FINISH_VISCHUNK(o0, f40, f42, g3)
finish_f42: FINISH_VISCHUNK(o0, f42, f44, g3)
finish_f44: FINISH_VISCHUNK(o0, f44, f46, g3)
finish_f46: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
vis_out_slk:
#ifdef __KERNEL__
srl asi_src, 3, %g5 ! IEU0 Group
xor asi_src, ASI_BLK_XOR1, asi_src ! IEU1
xor asi_src, %g5, asi_src ! IEU0 Group
#endif
vis_slk:ASI_SETSRC_NOBLK ! LSU Group
EXVIS4(LDDF [%o1] ASINORMAL, %f2) ! Load Group
add %o1, 8, %o1 ! IEU0
subcc %g3, 8, %g3 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f0, %f2, %f8 ! GRU Group
EXVIS5(STDF %f8, [%o0] ASINORMAL) ! Store
bl,pn %xcc, vis_out_slp ! CTI
add %o0, 8, %o0 ! IEU0 Group
ASI_SETSRC_NOBLK ! LSU Group
EXVIS4(LDDF [%o1] ASINORMAL, %f0) ! Load Group
add %o1, 8, %o1 ! IEU0
subcc %g3, 8, %g3 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f2, %f0, %f8 ! GRU Group
EXVIS5(STDF %f8, [%o0] ASINORMAL) ! Store
bge,pt %xcc, vis_slk ! CTI
add %o0, 8, %o0 ! IEU0 Group
vis_out_slp:
#ifdef __KERNEL__
brz,pt %o2, vis_ret ! CTI Group
mov %g1, %o1 ! IEU0
ba,pt %xcc, vis_slp+4 ! CTI Group
ASI_SETSRC_NOBLK ! LSU Group
#endif
vis_out:brz,pt %o2, vis_ret ! CTI Group
mov %g1, %o1 ! IEU0
#ifdef __KERNEL__
srl asi_src, 3, %g5 ! IEU0 Group
xor asi_src, ASI_BLK_XOR1, asi_src ! IEU1
xor asi_src, %g5, asi_src ! IEU0 Group
#endif
vis_slp:ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUB [%o1] ASINORMAL, %g5) ! LOAD
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
subcc %o2, 1, %o2 ! IEU1
bne,pt %xcc, vis_slp ! CTI
EX(STB %g5, [%o0 - 1] ASINORMAL,
add %o2, 1) ! Store Group
vis_ret:membar #StoreLoad | #StoreStore ! LSU Group
FPU_CLEAN_RETL
__memcpy_short:
andcc %o2, 1, %g0 ! IEU1 Group
be,pt %icc, 2f ! CTI
1: ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUB [%o1] ASINORMAL, %g5) ! LOAD Group
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
subcc %o2, 1, %o2 ! IEU1 Group
be,pn %xcc, short_ret ! CTI
EX(STB %g5, [%o0 - 1] ASINORMAL,
add %o2, 1) ! Store
2: ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUB [%o1] ASINORMAL, %g5) ! LOAD Group
add %o0, 2, %o0 ! IEU0
EX2(LDUB [%o1 + 1] ASINORMAL, %o5,
sub %o0, 2, %o0,
add %o2, %g0) ! LOAD Group
add %o1, 2, %o1 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
subcc %o2, 2, %o2 ! IEU1 Group
EX(STB %g5, [%o0 - 2] ASINORMAL,
add %o2, 2) ! Store
bne,pt %xcc, 2b ! CTI
EX(STB %o5, [%o0 - 1] ASINORMAL,
add %o2, 1) ! Store
short_ret:
NORMAL_RETL
#ifndef __KERNEL__
memcpy_private:
memcpy:
#ifndef REGS_64BIT
srl %o2, 0, %o2 ! IEU1 Group
#endif
brz,pn %o2, short_ret ! CTI Group
mov %o0, %g6 ! IEU0
#endif
__memcpy_entry:
cmp %o2, 15 ! IEU1 Group
bleu,pn %xcc, __memcpy_short ! CTI
cmp %o2, (64 * 6) ! IEU1 Group
bgeu,pn %xcc, VIS_enter ! CTI
#ifdef __KERNEL__
__memcpy_16plus:
#endif
andcc %o0, 7, %g2 ! IEU1 Group
sub %o0, %o1, %g5 ! IEU0
andcc %g5, 3, %o5 ! IEU1 Group
bne,pn %xcc, memcpy_noVIS_misaligned ! CTI
andcc %o1, 3, %g0 ! IEU1 Group
#ifdef REGS_64BIT
be,a,pt %xcc, 3f ! CTI
andcc %o1, 4, %g0 ! IEU1 Group
andcc %o1, 1, %g0 ! IEU1 Group
#else /* !REGS_64BIT */
be,pt %xcc, 5f ! CTI
andcc %o1, 1, %g0 ! IEU1 Group
#endif /* !REGS_64BIT */
be,pn %xcc, 4f ! CTI
andcc %o1, 2, %g0 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUB [%o1] ASINORMAL, %g2) ! Load Group
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
sub %o2, 1, %o2 ! IEU0 Group
ASI_SETDST_NOBLK ! LSU Group
bne,pn %xcc, 5f ! CTI Group
EX(STB %g2, [%o0 - 1] ASINORMAL,
add %o2, 1) ! Store
4: ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUH [%o1] ASINORMAL, %g2) ! Load Group
add %o1, 2, %o1 ! IEU0
add %o0, 2, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
sub %o2, 2, %o2 ! IEU0
EX(STH %g2, [%o0 - 2] ASINORMAL,
add %o2, 2) ! Store Group + bubble
#ifdef REGS_64BIT
5: andcc %o1, 4, %g0 ! IEU1
3: be,a,pn %xcc, 2f ! CTI
andcc %o2, -128, %g7 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EXO2(LDUW [%o1] ASINORMAL, %g5) ! Load Group
add %o1, 4, %o1 ! IEU0
add %o0, 4, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
sub %o2, 4, %o2 ! IEU0 Group
EX(STW %g5, [%o0 - 4] ASINORMAL,
add %o2, 4) ! Store
andcc %o2, -128, %g7 ! IEU1 Group
2: be,pn %xcc, 3f ! CTI
andcc %o0, 4, %g0 ! IEU1 Group
be,pn %xcc, 82f + 4 ! CTI Group
#else /* !REGS_64BIT */
5: andcc %o2, -128, %g7 ! IEU1
be,a,pn %xcc, 41f ! CTI
andcc %o2, 0x70, %g7 ! IEU1 Group
#endif /* !REGS_64BIT */
5: MOVE_BIGCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
MOVE_BIGCHUNK(o1, o0, 0x20, g1, g3, g5, o5)
MOVE_BIGCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
MOVE_BIGCHUNK(o1, o0, 0x60, g1, g3, g5, o5)
EXT(5b,35f,VIScopyfixup1)
35: subcc %g7, 128, %g7 ! IEU1 Group
add %o1, 128, %o1 ! IEU0
bne,pt %xcc, 5b ! CTI
add %o0, 128, %o0 ! IEU0 Group
3: andcc %o2, 0x70, %g7 ! IEU1 Group
41: be,pn %xcc, 80f ! CTI
andcc %o2, 8, %g0 ! IEU1 Group
#ifdef __KERNEL__
79: sethi %hi(80f), %o5 ! IEU0
sll %g7, 1, %g5 ! IEU0 Group
add %o1, %g7, %o1 ! IEU1
srl %g7, 1, %g2 ! IEU0 Group
sub %o5, %g5, %o5 ! IEU1
sub %o5, %g2, %o5 ! IEU0 Group
jmpl %o5 + %lo(80f), %g0 ! CTI Group brk forced
add %o0, %g7, %o0 ! IEU0 Group
#else
! Clk1 8-(
! Clk2 8-(
! Clk3 8-(
! Clk4 8-(
79: rd %pc, %o5 ! PDU Group
sll %g7, 1, %g5 ! IEU0 Group
add %o1, %g7, %o1 ! IEU1
sub %o5, %g5, %o5 ! IEU0 Group
jmpl %o5 + %lo(80f - 79b), %g0 ! CTI Group brk forced
add %o0, %g7, %o0 ! IEU0 Group
#endif
36: MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g5, o5)
MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g5, o5)
EXT(36b,80f,VIScopyfixup2)
80: be,pt %xcc, 81f ! CTI
andcc %o2, 4, %g0 ! IEU1
#ifdef REGS_64BIT
ASI_SETSRC_NOBLK ! LSU Group
EX(LDX [%o1] ASINORMAL, %g2,
and %o2, 0xf) ! Load Group
add %o0, 8, %o0 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
EX(STW %g2, [%o0 - 0x4] ASINORMAL,
and %o2, 0xf) ! Store Group
add %o1, 8, %o1 ! IEU1
srlx %g2, 32, %g2 ! IEU0 Group
EX2(STW %g2, [%o0 - 0x8] ASINORMAL,
and %o2, 0xf, %o2,
sub %o2, 4) ! Store
#else /* !REGS_64BIT */
lduw [%o1], %g2 ! Load Group
add %o0, 8, %o0 ! IEU0
lduw [%o1 + 0x4], %g3 ! Load Group
add %o1, 8, %o1 ! IEU0
stw %g2, [%o0 - 0x8] ! Store Group
stw %g3, [%o0 - 0x4] ! Store Group
#endif /* !REGS_64BIT */
81: be,pt %xcc, 1f ! CTI
andcc %o2, 2, %g0 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUW [%o1] ASINORMAL, %g2,
and %o2, 0x7) ! Load Group
add %o1, 4, %o1 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
EX(STW %g2, [%o0] ASINORMAL,
and %o2, 0x7) ! Store Group
add %o0, 4, %o0 ! IEU0
1: be,pt %xcc, 1f ! CTI
andcc %o2, 1, %g0 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUH [%o1] ASINORMAL, %g2,
and %o2, 0x3) ! Load Group
add %o1, 2, %o1 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
EX(STH %g2, [%o0] ASINORMAL,
and %o2, 0x3) ! Store Group
add %o0, 2, %o0 ! IEU0
1: be,pt %xcc, normal_retl ! CTI
nop ! IEU1
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUB [%o1] ASINORMAL, %g2,
add %g0, 1) ! Load Group
ASI_SETDST_NOBLK ! LSU Group
EX(STB %g2, [%o0] ASINORMAL,
add %g0, 1) ! Store Group + bubble
normal_retl:
NORMAL_RETL
#ifdef REGS_64BIT
82: MOVE_BIGALIGNCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
MOVE_BIGALIGNCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
EXT(82b,37f,VIScopyfixup3)
37: subcc %g7, 128, %g7 ! IEU1 Group
add %o1, 128, %o1 ! IEU0
bne,pt %xcc, 82b ! CTI
add %o0, 128, %o0 ! IEU0 Group
andcc %o2, 0x70, %g7 ! IEU1
be,pn %xcc, 84f ! CTI
andcc %o2, 8, %g0 ! IEU1 Group
#ifdef __KERNEL__
83: srl %g7, 1, %g5 ! IEU0
sethi %hi(84f), %o5 ! IEU0 Group
add %g7, %g5, %g5 ! IEU1
add %o1, %g7, %o1 ! IEU0 Group
sub %o5, %g5, %o5 ! IEU1
jmpl %o5 + %lo(84f), %g0 ! CTI Group brk forced
add %o0, %g7, %o0 ! IEU0 Group
#else
! Clk1 8-(
! Clk2 8-(
! Clk3 8-(
! Clk4 8-(
83: rd %pc, %o5 ! PDU Group
add %o1, %g7, %o1 ! IEU0 Group
sub %o5, %g7, %o5 ! IEU1
jmpl %o5 + %lo(84f - 83b), %g0 ! CTI Group brk forced
add %o0, %g7, %o0 ! IEU0 Group
#endif
38: MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3)
MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3)
EXT(38b,84f,VIScopyfixup4)
84: be,pt %xcc, 85f ! CTI Group
andcc %o2, 4, %g0 ! IEU1
ASI_SETSRC_NOBLK ! LSU Group
EX(LDX [%o1] ASINORMAL, %g2,
and %o2, 0xf) ! Load Group
add %o0, 8, %o0 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
add %o1, 8, %o1 ! IEU0 Group
EX(STX %g2, [%o0 - 0x8] ASINORMAL,
and %o2, 0xf) ! Store
85: be,pt %xcc, 1f ! CTI
andcc %o2, 2, %g0 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUW [%o1] ASINORMAL, %g2,
and %o2, 0x7) ! Load Group
add %o0, 4, %o0 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
add %o1, 4, %o1 ! IEU0 Group
EX(STW %g2, [%o0 - 0x4] ASINORMAL,
and %o2, 0x7) ! Store
1: be,pt %xcc, 1f ! CTI
andcc %o2, 1, %g0 ! IEU1 Group
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUH [%o1] ASINORMAL, %g2,
and %o2, 0x3) ! Load Group
add %o0, 2, %o0 ! IEU0
ASI_SETDST_NOBLK ! LSU Group
add %o1, 2, %o1 ! IEU0 Group
EX(STH %g2, [%o0 - 0x2] ASINORMAL,
and %o2, 0x3) ! Store
1: be,pt %xcc, 1f ! CTI
nop ! IEU0 Group
ASI_SETSRC_NOBLK ! LSU Group
EX(LDUB [%o1] ASINORMAL, %g2,
add %g0, 1) ! Load Group
ASI_SETDST_NOBLK ! LSU Group
EX(STB %g2, [%o0] ASINORMAL,
add %g0, 1) ! Store Group + bubble
1: NORMAL_RETL
#endif /* REGS_64BIT */
memcpy_noVIS_misaligned:
brz,pt %g2, 2f ! CTI Group
mov 8, %g1 ! IEU0
sub %g1, %g2, %g2 ! IEU0 Group
sub %o2, %g2, %o2 ! IEU0 Group
1: ASI_SETSRC_NOBLK ! LSU Group
EX(LDUB [%o1] ASINORMAL, %g5,
add %o2, %g2) ! Load Group
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
subcc %g2, 1, %g2 ! IEU1 Group
bne,pt %xcc, 1b ! CTI
EX2(STB %g5, [%o0 - 1] ASINORMAL,
add %o2, %g2, %o2,
add %o2, 1) ! Store
2:
#ifdef __KERNEL__
VISEntry
#endif
andn %o2, 7, %g5 ! IEU0 Group
and %o2, 7, %o2 ! IEU1
fmovd %f0, %f2 ! FPU
ASI_SETSRC_NOBLK ! LSU Group
alignaddr %o1, %g0, %g1 ! GRU Group
EXO2(LDDF [%g1] ASINORMAL, %f4) ! Load Group
1: EX(LDDF [%g1 + 0x8] ASINORMAL, %f6,
add %o2, %g5) ! Load Group
add %g1, 0x8, %g1 ! IEU0 Group
subcc %g5, 8, %g5 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f4, %f6, %f0 ! GRU Group
EX2(STDF %f0, [%o0] ASINORMAL,
add %o2, %g5, %o2,
add %o2, 8) ! Store
add %o1, 8, %o1 ! IEU0 Group
be,pn %xcc, end_cruft ! CTI
add %o0, 8, %o0 ! IEU1
ASI_SETSRC_NOBLK ! LSU Group
EX(LDDF [%g1 + 0x8] ASINORMAL, %f4,
add %o2, %g5) ! Load Group
add %g1, 8, %g1 ! IEU0
subcc %g5, 8, %g5 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
faligndata %f6, %f4, %f0 ! GRU Group
EX2(STDF %f0, [%o0] ASINORMAL,
add %o2, %g5, %o2,
add %o2, 8) ! Store
add %o1, 8, %o1 ! IEU0
ASI_SETSRC_NOBLK ! LSU Group
bne,pn %xcc, 1b ! CTI Group
add %o0, 8, %o0 ! IEU0
end_cruft:
brz,pn %o2, fpu_retl ! CTI Group
#ifndef __KERNEL__
nop ! IEU0
#else
ASI_SETSRC_NOBLK ! LSU Group
#endif
EXO2(LDUB [%o1] ASINORMAL, %g5) ! LOAD
add %o1, 1, %o1 ! IEU0
add %o0, 1, %o0 ! IEU1
ASI_SETDST_NOBLK ! LSU Group
subcc %o2, 1, %o2 ! IEU1
bne,pt %xcc, vis_slp ! CTI
EX(STB %g5, [%o0 - 1] ASINORMAL,
add %o2, 1) ! Store Group
fpu_retl:
FPU_RETL
#ifdef __KERNEL__
.section .fixup
.align 4
VIScopyfixup_reto2:
mov %o2, %o1
VIScopyfixup_ret:
/* If this is copy_from_user(), zero out the rest of the
* kernel buffer.
*/
ldub [%g6 + AOFF_task_tss + AOFF_thread_current_ds], %o4
andcc asi_src, 0x1, %g0
be,pt %icc, 1f
VISExit
andcc asi_dest, 0x1, %g0
bne,pn %icc, 1f
nop
save %sp, -160, %sp
mov %i0, %o0
call __bzero
mov %i1, %o1
restore
1: mov %o1, %o0
retl
wr %o4, %g0, %asi
VIScopyfixup1: subcc %g2, 18, %g2
add %o0, 32, %o0
bgeu,a,pt %icc, VIScopyfixup1
sub %g7, 32, %g7
sub %o0, 32, %o0
rd %pc, %g5
add %g2, (18 + 16), %g2
ldub [%g5 + %g2], %g2
ba,a,pt %xcc, 2f
.byte 0, 0, 0, 0, 0, 0, 0, 4, 4, 8, 12, 12, 16, 20, 20, 24, 28, 28
.align 4
VIScopyfixup2: mov (7 * 16), %g7
1: subcc %g2, 10, %g2
bgeu,a,pt %icc, 1b
sub %g7, 16, %g7
sub %o0, %g7, %o0
rd %pc, %g5
add %g2, (10 + 16), %g2
ldub [%g5 + %g2], %g2
ba,a,pt %xcc, 4f
.byte 0, 0, 0, 0, 0, 4, 4, 8, 12, 12
.align 4
VIScopyfixup3: subcc %g2, 10, %g2
add %o0, 32, %o0
bgeu,a,pt %icc, VIScopyfixup3
sub %g7, 32, %g7
sub %o0, 32, %o0
rd %pc, %g5
add %g2, (10 + 16), %g2
ldub [%g5 + %g2], %g2
ba,a,pt %xcc, 2f
.byte 0, 0, 0, 0, 0, 0, 0, 8, 16, 24
.align 4
2: and %o2, 0x7f, %o2
sub %g7, %g2, %g7
ba,pt %xcc, VIScopyfixup_ret
add %g7, %o2, %o1
VIScopyfixup4: mov (7 * 16), %g7
3: subcc %g2, 6, %g2
bgeu,a,pt %icc, 3b
sub %g7, 16, %g7
sub %o0, %g7, %o0
rd %pc, %g5
add %g2, (6 + 16), %g2
ldub [%g5 + %g2], %g2
ba,a,pt %xcc, 4f
.byte 0, 0, 0, 0, 0, 8
.align 4
4: and %o2, 0xf, %o2
sub %g7, %g2, %g7
ba,pt %xcc, VIScopyfixup_ret
add %g7, %o2, %o1
VIScopyfixup_vis3:
sub %o2, 0x80, %o2
VIScopyfixup_vis2:
add %o2, 0x40, %o2
VIScopyfixup_vis0:
add %o2, 0x80, %o2
VIScopyfixup_vis1:
add %g7, %g3, %g7
ba,pt %xcc, VIScopyfixup_ret
add %o2, %g7, %o1
VIScopyfixup_vis5:
add %g3, 8, %g3
VIScopyfixup_vis4:
add %g3, 8, %g3
ba,pt %xcc, VIScopyfixup_ret
add %o2, %g3, %o1
#endif
#ifdef __KERNEL__
.text
.align 32
.globl __memmove
.type __memmove,@function
.globl memmove
.type memmove,@function
memmove:
__memmove: cmp %o0, %o1
blu,pt %xcc, memcpy_private
sub %o0, %o1, %g5
add %o1, %o2, %g3
cmp %g3, %o0
bleu,pt %xcc, memcpy_private
add %o1, %o2, %g5
add %o0, %o2, %o5
sub %g5, 1, %o1
sub %o5, 1, %o0
1: ldub [%o1], %g5
subcc %o2, 1, %o2
sub %o1, 1, %o1
stb %g5, [%o0]
bne,pt %icc, 1b
sub %o0, 1, %o0
retl
clr %o0
#endif