source: src/linux/universal/linux-4.9/arch/parisc/lib/lusercopy.S @ 31885

Last change on this file since 31885 was 31885, checked in by brainslayer, 5 weeks ago

update

File size: 10.9 KB
Line 
1/*
2 *    User Space Access Routines
3 *
4 *    Copyright (C) 2000-2002 Hewlett-Packard (John Marvin)
5 *    Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org>
6 *    Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr>
7 *    Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org>
8 *    Copyright (C) 2017 Helge Deller <deller@gmx.de>
9 *    Copyright (C) 2017 John David Anglin <dave.anglin@bell.net>
10 *
11 *
12 *    This program is free software; you can redistribute it and/or modify
13 *    it under the terms of the GNU General Public License as published by
14 *    the Free Software Foundation; either version 2, or (at your option)
15 *    any later version.
16 *
17 *    This program is distributed in the hope that it will be useful,
18 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
19 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 *    GNU General Public License for more details.
21 *
22 *    You should have received a copy of the GNU General Public License
23 *    along with this program; if not, write to the Free Software
24 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25 */
26
27/*
28 * These routines still have plenty of room for optimization
29 * (word & doubleword load/store, dual issue, store hints, etc.).
30 */
31
32/*
33 * The following routines assume that space register 3 (sr3) contains
34 * the space id associated with the current users address space.
35 */
36
37
38        .text
39       
40#include <asm/assembly.h>
41#include <asm/errno.h>
42#include <linux/linkage.h>
43
44        /*
45         * get_sr gets the appropriate space value into
46         * sr1 for kernel/user space access, depending
47         * on the flag stored in the task structure.
48         */
49
50        .macro  get_sr
51        mfctl       %cr30,%r1
52        ldw         TI_SEGMENT(%r1),%r22
53        mfsp        %sr3,%r1
54        or,<>       %r22,%r0,%r0
55        copy        %r0,%r1
56        mtsp        %r1,%sr1
57        .endm
58
59        .macro fixup_branch lbl
60        ldil        L%\lbl, %r1
61        ldo         R%\lbl(%r1), %r1
62        bv          %r0(%r1)
63        .endm
64
65        /*
66         * unsigned long lclear_user(void *to, unsigned long n)
67         *
68         * Returns 0 for success.
69         * otherwise, returns number of bytes not transferred.
70         */
71
72ENTRY_CFI(lclear_user)
73        .proc
74        .callinfo NO_CALLS
75        .entry
76        comib,=,n   0,%r25,$lclu_done
77        get_sr
78$lclu_loop:
79        addib,<>    -1,%r25,$lclu_loop
801:      stbs,ma     %r0,1(%sr1,%r26)
81
82$lclu_done:
83        bv          %r0(%r2)
84        copy        %r25,%r28
85        .exit
86ENDPROC_CFI(lclear_user)
87
88        .section .fixup,"ax"
892:      fixup_branch $lclu_done
90        ldo        1(%r25),%r25
91        .previous
92
93        ASM_EXCEPTIONTABLE_ENTRY(1b,2b)
94
95        .procend
96
97        /*
98         * long lstrnlen_user(char *s, long n)
99         *
100         * Returns 0 if exception before zero byte or reaching N,
101         *         N+1 if N would be exceeded,
102         *         else strlen + 1 (i.e. includes zero byte).
103         */
104
105ENTRY_CFI(lstrnlen_user)
106        .proc
107        .callinfo NO_CALLS
108        .entry
109        comib,=     0,%r25,$lslen_nzero
110        copy        %r26,%r24
111        get_sr
1121:      ldbs,ma     1(%sr1,%r26),%r1
113$lslen_loop:
114        comib,=,n   0,%r1,$lslen_done
115        addib,<>    -1,%r25,$lslen_loop
1162:      ldbs,ma     1(%sr1,%r26),%r1
117$lslen_done:
118        bv          %r0(%r2)
119        sub         %r26,%r24,%r28
120        .exit
121
122$lslen_nzero:
123        b           $lslen_done
124        ldo         1(%r26),%r26 /* special case for N == 0 */
125ENDPROC_CFI(lstrnlen_user)
126
127        .section .fixup,"ax"
1283:      fixup_branch $lslen_done
129        copy        %r24,%r26    /* reset r26 so 0 is returned on fault */
130        .previous
131
132        ASM_EXCEPTIONTABLE_ENTRY(1b,3b)
133        ASM_EXCEPTIONTABLE_ENTRY(2b,3b)
134
135        .procend
136
137
138
139/*
140 * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len)
141 *
142 * Inputs:
143 * - sr1 already contains space of source region
144 * - sr2 already contains space of destination region
145 *
146 * Returns:
147 * - number of bytes that could not be copied.
148 *   On success, this will be zero.
149 *
150 * This code is based on a C-implementation of a copy routine written by
151 * Randolph Chung, which in turn was derived from the glibc.
152 *
153 * Several strategies are tried to try to get the best performance for various
154 * conditions. In the optimal case, we copy by loops that copy 32- or 16-bytes
155 * at a time using general registers.  Unaligned copies are handled either by
156 * aligning the destination and then using shift-and-write method, or in a few
157 * cases by falling back to a byte-at-a-time copy.
158 *
159 * Testing with various alignments and buffer sizes shows that this code is
160 * often >10x faster than a simple byte-at-a-time copy, even for strangely
161 * aligned operands. It is interesting to note that the glibc version of memcpy
162 * (written in C) is actually quite fast already. This routine is able to beat
163 * it by 30-40% for aligned copies because of the loop unrolling, but in some
164 * cases the glibc version is still slightly faster. This lends more
165 * credibility that gcc can generate very good code as long as we are careful.
166 *
167 * Possible optimizations:
168 * - add cache prefetching
169 * - try not to use the post-increment address modifiers; they may create
170 *   additional interlocks. Assumption is that those were only efficient on old
171 *   machines (pre PA8000 processors)
172 */
173
174        dst = arg0
175        src = arg1
176        len = arg2
177        end = arg3
178        t1  = r19
179        t2  = r20
180        t3  = r21
181        t4  = r22
182        srcspc = sr1
183        dstspc = sr2
184
185        t0 = r1
186        a1 = t1
187        a2 = t2
188        a3 = t3
189        a0 = t4
190
191        save_src = ret0
192        save_dst = ret1
193        save_len = r31
194
195ENTRY_CFI(pa_memcpy)
196        .proc
197        .callinfo NO_CALLS
198        .entry
199
200        /* Last destination address */
201        add     dst,len,end
202
203        /* short copy with less than 16 bytes? */
204        cmpib,COND(>>=),n 15,len,.Lbyte_loop
205
206        /* same alignment? */
207        xor     src,dst,t0
208        extru   t0,31,2,t1
209        cmpib,<>,n  0,t1,.Lunaligned_copy
210
211#ifdef CONFIG_64BIT
212        /* only do 64-bit copies if we can get aligned. */
213        extru   t0,31,3,t1
214        cmpib,<>,n  0,t1,.Lalign_loop32
215
216        /* loop until we are 64-bit aligned */
217.Lalign_loop64:
218        extru   dst,31,3,t1
219        cmpib,=,n       0,t1,.Lcopy_loop_16_start
22020:     ldb,ma  1(srcspc,src),t1
22121:     stb,ma  t1,1(dstspc,dst)
222        b       .Lalign_loop64
223        ldo     -1(len),len
224
225        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
226        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
227
228.Lcopy_loop_16_start:
229        ldi     31,t0
230.Lcopy_loop_16:
231        cmpb,COND(>>=),n t0,len,.Lword_loop
232
23310:     ldd     0(srcspc,src),t1
23411:     ldd     8(srcspc,src),t2
235        ldo     16(src),src
23612:     std,ma  t1,8(dstspc,dst)
23713:     std,ma  t2,8(dstspc,dst)
23814:     ldd     0(srcspc,src),t1
23915:     ldd     8(srcspc,src),t2
240        ldo     16(src),src
24116:     std,ma  t1,8(dstspc,dst)
24217:     std,ma  t2,8(dstspc,dst)
243
244        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
245        ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy16_fault)
246        ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
247        ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
248        ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
249        ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy16_fault)
250        ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
251        ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
252
253        b       .Lcopy_loop_16
254        ldo     -32(len),len
255
256.Lword_loop:
257        cmpib,COND(>>=),n 3,len,.Lbyte_loop
25820:     ldw,ma  4(srcspc,src),t1
25921:     stw,ma  t1,4(dstspc,dst)
260        b       .Lword_loop
261        ldo     -4(len),len
262
263        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
264        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
265
266#endif /* CONFIG_64BIT */
267
268        /* loop until we are 32-bit aligned */
269.Lalign_loop32:
270        extru   dst,31,2,t1
271        cmpib,=,n       0,t1,.Lcopy_loop_8
27220:     ldb,ma  1(srcspc,src),t1
27321:     stb,ma  t1,1(dstspc,dst)
274        b       .Lalign_loop32
275        ldo     -1(len),len
276
277        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
278        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
279
280
281.Lcopy_loop_8:
282        cmpib,COND(>>=),n 15,len,.Lbyte_loop
283
28410:     ldw     0(srcspc,src),t1
28511:     ldw     4(srcspc,src),t2
28612:     stw,ma  t1,4(dstspc,dst)
28713:     stw,ma  t2,4(dstspc,dst)
28814:     ldw     8(srcspc,src),t1
28915:     ldw     12(srcspc,src),t2
290        ldo     16(src),src
29116:     stw,ma  t1,4(dstspc,dst)
29217:     stw,ma  t2,4(dstspc,dst)
293
294        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
295        ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy8_fault)
296        ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
297        ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
298        ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
299        ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy8_fault)
300        ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
301        ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
302
303        b       .Lcopy_loop_8
304        ldo     -16(len),len
305
306.Lbyte_loop:
307        cmpclr,COND(<>) len,%r0,%r0
308        b,n     .Lcopy_done
30920:     ldb     0(srcspc,src),t1
310        ldo     1(src),src
31121:     stb,ma  t1,1(dstspc,dst)
312        b       .Lbyte_loop
313        ldo     -1(len),len
314
315        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
316        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
317
318.Lcopy_done:
319        bv      %r0(%r2)
320        sub     end,dst,ret0
321
322
323        /* src and dst are not aligned the same way. */
324        /* need to go the hard way */
325.Lunaligned_copy:
326        /* align until dst is 32bit-word-aligned */
327        extru   dst,31,2,t1
328        cmpib,=,n       0,t1,.Lcopy_dstaligned
32920:     ldb     0(srcspc,src),t1
330        ldo     1(src),src
33121:     stb,ma  t1,1(dstspc,dst)
332        b       .Lunaligned_copy
333        ldo     -1(len),len
334
335        ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
336        ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
337
338.Lcopy_dstaligned:
339
340        /* store src, dst and len in safe place */
341        copy    src,save_src
342        copy    dst,save_dst
343        copy    len,save_len
344
345        /* len now needs give number of words to copy */
346        SHRREG  len,2,len
347
348        /*
349         * Copy from a not-aligned src to an aligned dst using shifts.
350         * Handles 4 words per loop.
351         */
352
353        depw,z src,28,2,t0
354        subi 32,t0,t0
355        mtsar t0
356        extru len,31,2,t0
357        cmpib,= 2,t0,.Lcase2
358        /* Make src aligned by rounding it down.  */
359        depi 0,31,2,src
360
361        cmpiclr,<> 3,t0,%r0
362        b,n .Lcase3
363        cmpiclr,<> 1,t0,%r0
364        b,n .Lcase1
365.Lcase0:
366        cmpb,COND(=) %r0,len,.Lcda_finish
367        nop
368
3691:      ldw,ma 4(srcspc,src), a3
370        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
3711:      ldw,ma 4(srcspc,src), a0
372        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
373        b,n .Ldo3
374.Lcase1:
3751:      ldw,ma 4(srcspc,src), a2
376        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
3771:      ldw,ma 4(srcspc,src), a3
378        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
379        ldo -1(len),len
380        cmpb,COND(=),n %r0,len,.Ldo0
381.Ldo4:
3821:      ldw,ma 4(srcspc,src), a0
383        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
384        shrpw a2, a3, %sar, t0
3851:      stw,ma t0, 4(dstspc,dst)
386        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
387.Ldo3:
3881:      ldw,ma 4(srcspc,src), a1
389        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
390        shrpw a3, a0, %sar, t0
3911:      stw,ma t0, 4(dstspc,dst)
392        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
393.Ldo2:
3941:      ldw,ma 4(srcspc,src), a2
395        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
396        shrpw a0, a1, %sar, t0
3971:      stw,ma t0, 4(dstspc,dst)
398        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
399.Ldo1:
4001:      ldw,ma 4(srcspc,src), a3
401        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
402        shrpw a1, a2, %sar, t0
4031:      stw,ma t0, 4(dstspc,dst)
404        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
405        ldo -4(len),len
406        cmpb,COND(<>) %r0,len,.Ldo4
407        nop
408.Ldo0:
409        shrpw a2, a3, %sar, t0
4101:      stw,ma t0, 4(dstspc,dst)
411        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
412
413.Lcda_rdfault:
414.Lcda_finish:
415        /* calculate new src, dst and len and jump to byte-copy loop */
416        sub     dst,save_dst,t0
417        add     save_src,t0,src
418        b       .Lbyte_loop
419        sub     save_len,t0,len
420
421.Lcase3:
4221:      ldw,ma 4(srcspc,src), a0
423        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
4241:      ldw,ma 4(srcspc,src), a1
425        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
426        b .Ldo2
427        ldo 1(len),len
428.Lcase2:
4291:      ldw,ma 4(srcspc,src), a1
430        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
4311:      ldw,ma 4(srcspc,src), a2
432        ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
433        b .Ldo1
434        ldo 2(len),len
435
436
437        /* fault exception fixup handlers: */
438#ifdef CONFIG_64BIT
439.Lcopy16_fault:
440        b       .Lcopy_done
44110:     std,ma  t1,8(dstspc,dst)
442        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
443#endif
444
445.Lcopy8_fault:
446        b       .Lcopy_done
44710:     stw,ma  t1,4(dstspc,dst)
448        ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
449
450        .exit
451ENDPROC_CFI(pa_memcpy)
452        .procend
453
454        .end
Note: See TracBrowser for help on using the repository browser.