@matheusbsbs / matheus-bagliano / commits / 84702995f8
Fork 0

A better-scheduled PPC SHA-1 implementation.

This is about 15% faster that the current sha1ppc.S on a G4, and 5% faster on a G5 when hashing 10 million bytes, unaligned. (The G5 ratio seems to get better as the sizes fall.) It's also somewhat smaller, due to using load-multiple instructions. No copyright is claimed on the changes to Paul Mackerras' work below.

Unknown committed Jun 24, 2006 at 02:31 UTC 84702995f89361c5872029cb93983c704d80e993
1 file changed +195 -156
ppc/sha1ppc.S
+195 -156
@@ -3,183 +3,222 @@
3 *
4 * Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
5 */
6 -#define FS 80
6
7 /*
9 - * We roll the registers for T, A, B, C, D, E around on each
10 - * iteration; T on iteration t is A on iteration t+1, and so on.
11 - * We use registers 7 - 12 for this.
8 + * PowerPC calling convention:
9 + * %r0 - volatile temp
10 + * %r1 - stack pointer.
11 + * %r2 - reserved
12 + * %r3-%r12 - Incoming arguments & return values; volatile.
13 + * %r13-%r31 - Callee-save registers
14 + * %lr - Return address, volatile
15 + * %ctr - volatile
16 + *
17 + * Register usage in this routine:
18 + * %r0 - temp
19 + * %r3 - argument (pointer to 5 words of SHA state)
20 + * %r4 - argument (pointer to data to hash)
21 + * %r5 - Contant K in SHA round (initially number of blocks to hash)
22 + * %r6-%r10 - Working copies of SHA variables A..E (actually E..A order)
23 + * %r11-%r26 - Data being hashed W[].
24 + * %r27-%r31 - Previous copies of A..E, for final add back.
25 + * %ctr - loop count
26 + */
27 +
28 +
29 +/*
30 + * We roll the registers for A, B, C, D, E around on each
31 + * iteration; E on iteration t is D on iteration t+1, and so on.
32 + * We use registers 6 - 10 for this. (Registers 27 - 31 hold
33 + * the previous values.)
34 */
13 -#define RT(t) ((((t)+5)%6)+7)
14 -#define RA(t) ((((t)+4)%6)+7)
15 -#define RB(t) ((((t)+3)%6)+7)
16 -#define RC(t) ((((t)+2)%6)+7)
17 -#define RD(t) ((((t)+1)%6)+7)
18 -#define RE(t) ((((t)+0)%6)+7)
19 -
20 -/* We use registers 16 - 31 for the W values */
21 -#define W(t) (((t)%16)+16)
22 -
23 -#define STEPD0(t) \
24 - and %r6,RB(t),RC(t); \
25 - andc %r0,RD(t),RB(t); \
26 - rotlwi RT(t),RA(t),5; \
27 - rotlwi RB(t),RB(t),30; \
28 - or %r6,%r6,%r0; \
29 - add %r0,RE(t),%r15; \
30 - add RT(t),RT(t),%r6; \
31 - add %r0,%r0,W(t); \
32 - add RT(t),RT(t),%r0
33 -
34 -#define STEPD1(t) \
35 - xor %r6,RB(t),RC(t); \
36 - rotlwi RT(t),RA(t),5; \
37 - rotlwi RB(t),RB(t),30; \
38 - xor %r6,%r6,RD(t); \
39 - add %r0,RE(t),%r15; \
40 - add RT(t),RT(t),%r6; \
41 - add %r0,%r0,W(t); \
42 - add RT(t),RT(t),%r0
43 -
44 -#define STEPD2(t) \
45 - and %r6,RB(t),RC(t); \
46 - and %r0,RB(t),RD(t); \
47 - rotlwi RT(t),RA(t),5; \
48 - rotlwi RB(t),RB(t),30; \
49 - or %r6,%r6,%r0; \
50 - and %r0,RC(t),RD(t); \
51 - or %r6,%r6,%r0; \
52 - add %r0,RE(t),%r15; \
53 - add RT(t),RT(t),%r6; \
54 - add %r0,%r0,W(t); \
55 - add RT(t),RT(t),%r0
56 -
57 -#define LOADW(t) \
58 - lwz W(t),(t)*4(%r4)
59 -
60 -#define UPDATEW(t) \
61 - xor %r0,W((t)-3),W((t)-8); \
62 - xor W(t),W((t)-16),W((t)-14); \
63 - xor W(t),W(t),%r0; \
64 - rotlwi W(t),W(t),1
65 -
66 -#define STEP0LD4(t) \
67 - STEPD0(t); LOADW((t)+4); \
68 - STEPD0((t)+1); LOADW((t)+5); \
69 - STEPD0((t)+2); LOADW((t)+6); \
70 - STEPD0((t)+3); LOADW((t)+7)
71 -
72 -#define STEPUP4(t, fn) \
73 - STEP##fn(t); UPDATEW((t)+4); \
74 - STEP##fn((t)+1); UPDATEW((t)+5); \
75 - STEP##fn((t)+2); UPDATEW((t)+6); \
76 - STEP##fn((t)+3); UPDATEW((t)+7)
77 -
78 -#define STEPUP20(t, fn) \
79 - STEPUP4(t, fn); \
80 - STEPUP4((t)+4, fn); \
81 - STEPUP4((t)+8, fn); \
82 - STEPUP4((t)+12, fn); \
83 - STEPUP4((t)+16, fn)
35 +#define RA(t) (((t)+4)%5+6)
36 +#define RB(t) (((t)+3)%5+6)
37 +#define RC(t) (((t)+2)%5+6)
38 +#define RD(t) (((t)+1)%5+6)
39 +#define RE(t) (((t)+0)%5+6)
40 +
41 +/* We use registers 11 - 26 for the W values */
42 +#define W(t) ((t)%16+11)
43 +
44 +/* Register 5 is used for the constant k */
45 +
46 +/*
47 + * The basic SHA-1 round function is:
48 + * E += ROTL(A,5) + F(B,C,D) + W[i] + K; B = ROTL(B,30)
49 + * Then the variables are renamed: (A,B,C,D,E) = (E,A,B,C,D).
50 + *
51 + * Every 20 rounds, the function F() and the contant K changes:
52 + * - 20 rounds of f0(b,c,d) = "bit wise b ? c : d" = (^b & d) + (b & c)
53 + * - 20 rounds of f1(b,c,d) = b^c^d = (b^d)^c
54 + * - 20 rounds of f2(b,c,d) = majority(b,c,d) = (b&d) + ((b^d)&c)
55 + * - 20 more rounds of f1(b,c,d)
56 + *
57 + * These are all scheduled for near-optimal performance on a G4.
58 + * The G4 is a 3-issue out-of-order machine with 3 ALUs, but it can only
59 + * *consider* starting the oldest 3 instructions per cycle. So to get
60 + * maximum performace out of it, you have to treat it as an in-order
61 + * machine. Which means interleaving the computation round t with the
62 + * computation of W[t+4].
63 + *
64 + * The first 16 rounds use W values loaded directly from memory, while the
65 + * remianing 64 use values computed from those first 16. We preload
66 + * 4 values before starting, so there are three kinds of rounds:
67 + * - The first 12 (all f0) also load the W values from memory.
68 + * - The next 64 compute W(i+4) in parallel. 8*f0, 20*f1, 20*f2, 16*f1.
69 + * - The last 4 (all f1) do not do anything with W.
70 + *
71 + * Therefore, we have 6 different round functions:
72 + * STEPD0_LOAD(t,s) - Perform round t and load W(s). s < 16
73 + * STEPD0_UPDATE(t,s) - Perform round t and compute W(s). s >= 16.
74 + * STEPD1_UPDATE(t,s)
75 + * STEPD2_UPDATE(t,s)
76 + * STEPD1(t) - Perform round t with no load or update.
77 + *
78 + * The G5 is more fully out-of-order, and can find the parallelism
79 + * by itself. The big limit is that it has a 2-cycle ALU latency, so
80 + * even though it's 2-way, the code has to be scheduled as if it's
81 + * 4-way, which can be a limit. To help it, we try to schedule the
82 + * read of RA(t) as late as possible so it doesn't stall waiting for
83 + * the previous round's RE(t-1), and we try to rotate RB(t) as early
84 + * as possible while reading RC(t) (= RB(t-1)) as late as possible.
85 + */
86 +
87 +/* the initial loads. */
88 +#define LOADW(s) \
89 + lwz W(s),(s)*4(%r4)
90 +
91 +/*
92 + * Perform a step with F0, and load W(s). Uses W(s) as a temporary
93 + * before loading it.
94 + * This is actually 10 instructions, which is an awkward fit.
95 + * It can execute grouped as listed, or delayed one instruction.
96 + * (If delayed two instructions, there is a stall before the start of the
97 + * second line.) Thus, two iterations take 7 cycles, 3.5 cycles per round.
98 + */
99 +#define STEPD0_LOAD(t,s) \
100 +add RE(t),RE(t),W(t); andc %r0,RD(t),RB(t); and W(s),RC(t),RB(t); \
101 +add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; rotlwi RB(t),RB(t),30; \
102 +add RE(t),RE(t),W(s); add %r0,%r0,%r5; lwz W(s),(s)*4(%r4); \
103 +add RE(t),RE(t),%r0
104 +
105 +/*
106 + * This is likewise awkward, 13 instructions. However, it can also
107 + * execute starting with 2 out of 3 possible moduli, so it does 2 rounds
108 + * in 9 cycles, 4.5 cycles/round.
109 + */
110 +#define STEPD0_UPDATE(t,s,loadk...) \
111 +add RE(t),RE(t),W(t); andc %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
112 +add RE(t),RE(t),%r0; and %r0,RC(t),RB(t); xor W(s),W(s),W((s)-8); \
113 +add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; xor W(s),W(s),W((s)-14); \
114 +add RE(t),RE(t),%r5; loadk; rotlwi RB(t),RB(t),30; rotlwi W(s),W(s),1; \
115 +add RE(t),RE(t),%r0
116 +
117 +/* Nicely optimal. Conveniently, also the most common. */
118 +#define STEPD1_UPDATE(t,s,loadk...) \
119 +add RE(t),RE(t),W(t); xor %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
120 +add RE(t),RE(t),%r5; loadk; xor %r0,%r0,RC(t); xor W(s),W(s),W((s)-8); \
121 +add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; xor W(s),W(s),W((s)-14); \
122 +add RE(t),RE(t),%r0; rotlwi RB(t),RB(t),30; rotlwi W(s),W(s),1
123 +
124 +/*
125 + * The naked version, no UPDATE, for the last 4 rounds. 3 cycles per.
126 + * We could use W(s) as a temp register, but we don't need it.
127 + */
128 +#define STEPD1(t) \
129 + add RE(t),RE(t),W(t); xor %r0,RD(t),RB(t); \
130 +rotlwi RB(t),RB(t),30; add RE(t),RE(t),%r5; xor %r0,%r0,RC(t); \
131 +add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; /* spare slot */ \
132 +add RE(t),RE(t),%r0
133 +
134 +/*
135 + * 14 instructions, 5 cycles per. The majority function is a bit
136 + * awkward to compute. This can execute with a 1-instruction delay,
137 + * but it causes a 2-instruction delay, which triggers a stall.
138 + */
139 +#define STEPD2_UPDATE(t,s,loadk...) \
140 +add RE(t),RE(t),W(t); and %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
141 +add RE(t),RE(t),%r0; xor %r0,RD(t),RB(t); xor W(s),W(s),W((s)-8); \
142 +add RE(t),RE(t),%r5; loadk; and %r0,%r0,RC(t); xor W(s),W(s),W((s)-14); \
143 +add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; rotlwi W(s),W(s),1; \
144 +add RE(t),RE(t),%r0; rotlwi RB(t),RB(t),30
145 +
146 +#define STEP0_LOAD4(t,s) \
147 + STEPD0_LOAD(t,s); \
148 + STEPD0_LOAD((t+1),(s)+1); \
149 + STEPD0_LOAD((t)+2,(s)+2); \
150 + STEPD0_LOAD((t)+3,(s)+3)
151 +
152 +#define STEPUP4(fn, t, s, loadk...) \
153 + STEP##fn##_UPDATE(t,s,); \
154 + STEP##fn##_UPDATE((t)+1,(s)+1,); \
155 + STEP##fn##_UPDATE((t)+2,(s)+2,); \
156 + STEP##fn##_UPDATE((t)+3,(s)+3,loadk)
157 +
158 +#define STEPUP20(fn, t, s, loadk...) \
159 + STEPUP4(fn, t, s,); \
160 + STEPUP4(fn, (t)+4, (s)+4,); \
161 + STEPUP4(fn, (t)+8, (s)+8,); \
162 + STEPUP4(fn, (t)+12, (s)+12,); \
163 + STEPUP4(fn, (t)+16, (s)+16, loadk)
164
165 .globl sha1_core
166 sha1_core:
87 - stwu %r1,-FS(%r1)
88 - stw %r15,FS-68(%r1)
89 - stw %r16,FS-64(%r1)
90 - stw %r17,FS-60(%r1)
91 - stw %r18,FS-56(%r1)
92 - stw %r19,FS-52(%r1)
93 - stw %r20,FS-48(%r1)
94 - stw %r21,FS-44(%r1)
95 - stw %r22,FS-40(%r1)
96 - stw %r23,FS-36(%r1)
97 - stw %r24,FS-32(%r1)
98 - stw %r25,FS-28(%r1)
99 - stw %r26,FS-24(%r1)
100 - stw %r27,FS-20(%r1)
101 - stw %r28,FS-16(%r1)
102 - stw %r29,FS-12(%r1)
103 - stw %r30,FS-8(%r1)
104 - stw %r31,FS-4(%r1)
167 + stwu %r1,-80(%r1)
168 + stmw %r13,4(%r1)
169
170 /* Load up A - E */
107 - lwz RA(0),0(%r3) /* A */
108 - lwz RB(0),4(%r3) /* B */
109 - lwz RC(0),8(%r3) /* C */
110 - lwz RD(0),12(%r3) /* D */
111 - lwz RE(0),16(%r3) /* E */
171 + lmw %r27,0(%r3)
172
173 mtctr %r5
174
115 -1: LOADW(0)
175 +1:
176 + LOADW(0)
177 + lis %r5,0x5a82
178 + mr RE(0),%r31
179 LOADW(1)
180 + mr RD(0),%r30
181 + mr RC(0),%r29
182 LOADW(2)
183 + ori %r5,%r5,0x7999 /* K0-19 */
184 + mr RB(0),%r28
185 LOADW(3)
186 + mr RA(0),%r27
187 +
188 + STEP0_LOAD4(0, 4)
189 + STEP0_LOAD4(4, 8)
190 + STEP0_LOAD4(8, 12)
191 + STEPUP4(D0, 12, 16,)
192 + STEPUP4(D0, 16, 20, lis %r5,0x6ed9)
193
120 - lis %r15,0x5a82 /* K0-19 */
121 - ori %r15,%r15,0x7999
122 - STEP0LD4(0)
123 - STEP0LD4(4)
124 - STEP0LD4(8)
125 - STEPUP4(12, D0)
126 - STEPUP4(16, D0)
127 -
128 - lis %r15,0x6ed9 /* K20-39 */
129 - ori %r15,%r15,0xeba1
130 - STEPUP20(20, D1)
131 -
132 - lis %r15,0x8f1b /* K40-59 */
133 - ori %r15,%r15,0xbcdc
134 - STEPUP20(40, D2)
135 -
136 - lis %r15,0xca62 /* K60-79 */
137 - ori %r15,%r15,0xc1d6
138 - STEPUP4(60, D1)
139 - STEPUP4(64, D1)
140 - STEPUP4(68, D1)
141 - STEPUP4(72, D1)
194 + ori %r5,%r5,0xeba1 /* K20-39 */
195 + STEPUP20(D1, 20, 24, lis %r5,0x8f1b)
196 +
197 + ori %r5,%r5,0xbcdc /* K40-59 */
198 + STEPUP20(D2, 40, 44, lis %r5,0xca62)
199 +
200 + ori %r5,%r5,0xc1d6 /* K60-79 */
201 + STEPUP4(D1, 60, 64,)
202 + STEPUP4(D1, 64, 68,)
203 + STEPUP4(D1, 68, 72,)
204 + STEPUP4(D1, 72, 76,)
205 + addi %r4,%r4,64
206 STEPD1(76)
207 STEPD1(77)
208 STEPD1(78)
209 STEPD1(79)
210
147 - lwz %r20,16(%r3)
148 - lwz %r19,12(%r3)
149 - lwz %r18,8(%r3)
150 - lwz %r17,4(%r3)
151 - lwz %r16,0(%r3)
152 - add %r20,RE(80),%r20
153 - add RD(0),RD(80),%r19
154 - add RC(0),RC(80),%r18
155 - add RB(0),RB(80),%r17
156 - add RA(0),RA(80),%r16
157 - mr RE(0),%r20
158 - stw RA(0),0(%r3)
159 - stw RB(0),4(%r3)
160 - stw RC(0),8(%r3)
161 - stw RD(0),12(%r3)
162 - stw RE(0),16(%r3)
211 + /* Add results to original values */
212 + add %r31,%r31,RE(0)
213 + add %r30,%r30,RD(0)
214 + add %r29,%r29,RC(0)
215 + add %r28,%r28,RB(0)
216 + add %r27,%r27,RA(0)
217
164 - addi %r4,%r4,64
218 bdnz 1b
219
167 - lwz %r15,FS-68(%r1)
168 - lwz %r16,FS-64(%r1)
169 - lwz %r17,FS-60(%r1)
170 - lwz %r18,FS-56(%r1)
171 - lwz %r19,FS-52(%r1)
172 - lwz %r20,FS-48(%r1)
173 - lwz %r21,FS-44(%r1)
174 - lwz %r22,FS-40(%r1)
175 - lwz %r23,FS-36(%r1)
176 - lwz %r24,FS-32(%r1)
177 - lwz %r25,FS-28(%r1)
178 - lwz %r26,FS-24(%r1)
179 - lwz %r27,FS-20(%r1)
180 - lwz %r28,FS-16(%r1)
181 - lwz %r29,FS-12(%r1)
182 - lwz %r30,FS-8(%r1)
183 - lwz %r31,FS-4(%r1)
184 - addi %r1,%r1,FS
220 + /* Save final hash, restore registers, and return */
221 + stmw %r27,0(%r3)
222 + lmw %r13,4(%r1)
223 + addi %r1,%r1,80
224 blr