1 // Copyright 2026 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "textflag.h"
6
7 // func MemHash32(p unsafe.Pointer, h uintptr) uintptr
8 TEXT ·MemHash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
9 MOVB ·UseAeshash(SB), R10
10 CBZ R10, noaes
11 MOVD $·aeskeysched+0(SB), R3
12
13 VEOR V0.B16, V0.B16, V0.B16
14 VLD1 (R3), [V2.B16]
15 VLD1 (R0), V0.S[2]
16 VMOV R1, V0.D[0]
17
18 AESE V2.B16, V0.B16
19 AESMC V0.B16, V0.B16
20 AESE V2.B16, V0.B16
21 AESMC V0.B16, V0.B16
22 AESE V2.B16, V0.B16
23
24 VMOV V0.D[0], R0
25 RET
26 noaes:
27 B ·memHash32Fallback<ABIInternal>(SB)
28
29 // func MemHash64(p unsafe.Pointer, h uintptr) uintptr
30 TEXT ·MemHash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
31 MOVB ·UseAeshash(SB), R10
32 CBZ R10, noaes
33 MOVD $·aeskeysched+0(SB), R3
34
35 VEOR V0.B16, V0.B16, V0.B16
36 VLD1 (R3), [V2.B16]
37 VLD1 (R0), V0.D[1]
38 VMOV R1, V0.D[0]
39
40 AESE V2.B16, V0.B16
41 AESMC V0.B16, V0.B16
42 AESE V2.B16, V0.B16
43 AESMC V0.B16, V0.B16
44 AESE V2.B16, V0.B16
45
46 VMOV V0.D[0], R0
47 RET
48 noaes:
49 B ·memHash64Fallback<ABIInternal>(SB)
50
51 // func MemHash(p unsafe.Pointer, h, size uintptr) uintptr
52 TEXT ·MemHash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
53 MOVB ·UseAeshash(SB), R10
54 CBZ R10, noaes
55 B ·aeshashbody<>(SB)
56 noaes:
57 B ·memHashFallback<ABIInternal>(SB)
58
59 // func StrHash(p unsafe.Pointer, h uintptr) uintptr
60 TEXT ·StrHash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
61 MOVB ·UseAeshash(SB), R10
62 CBZ R10, noaes
63 LDP (R0), (R0, R2) // string data / length
64 B ·aeshashbody<>(SB)
65 noaes:
66 B ·strHashFallback<ABIInternal>(SB)
67
68 // R0: data
69 // R1: seed data
70 // R2: length
71 // At return, R0 = return value
72 TEXT ·aeshashbody<>(SB),NOSPLIT|NOFRAME,$0
73 VEOR V30.B16, V30.B16, V30.B16
74 VMOV R1, V30.D[0]
75 VMOV R2, V30.D[1] // load length into seed
76
77 MOVD $·aeskeysched+0(SB), R4
78 VLD1.P 16(R4), [V0.B16]
79 AESE V30.B16, V0.B16
80 AESMC V0.B16, V0.B16
81 CMP $16, R2
82 BLO aes0to15
83 BEQ aes16
84 CMP $32, R2
85 BLS aes17to32
86 CMP $64, R2
87 BLS aes33to64
88 CMP $128, R2
89 BLS aes65to128
90 B aes129plus
91
92 aes0to15:
93 CBZ R2, aes0
94 VEOR V2.B16, V2.B16, V2.B16
95 TBZ $3, R2, less_than_8
96 VLD1.P 8(R0), V2.D[0]
97
98 less_than_8:
99 TBZ $2, R2, less_than_4
100 VLD1.P 4(R0), V2.S[2]
101
102 less_than_4:
103 TBZ $1, R2, less_than_2
104 VLD1.P 2(R0), V2.H[6]
105
106 less_than_2:
107 TBZ $0, R2, done
108 VLD1 (R0), V2.B[14]
109 done:
110 AESE V0.B16, V2.B16
111 AESMC V2.B16, V2.B16
112 AESE V0.B16, V2.B16
113 AESMC V2.B16, V2.B16
114 AESE V0.B16, V2.B16
115 AESMC V2.B16, V2.B16
116
117 VMOV V2.D[0], R0
118 RET
119
120 aes0:
121 VMOV V0.D[0], R0
122 RET
123
124 aes16:
125 VLD1 (R0), [V2.B16]
126 B done
127
128 aes17to32:
129 // make second seed
130 VLD1 (R4), [V1.B16]
131 AESE V30.B16, V1.B16
132 AESMC V1.B16, V1.B16
133 SUB $16, R2, R10
134 VLD1.P (R0)(R10), [V2.B16]
135 VLD1 (R0), [V3.B16]
136
137 AESE V0.B16, V2.B16
138 AESMC V2.B16, V2.B16
139 AESE V1.B16, V3.B16
140 AESMC V3.B16, V3.B16
141
142 AESE V0.B16, V2.B16
143 AESMC V2.B16, V2.B16
144 AESE V1.B16, V3.B16
145 AESMC V3.B16, V3.B16
146
147 AESE V0.B16, V2.B16
148 AESE V1.B16, V3.B16
149
150 VEOR V3.B16, V2.B16, V2.B16
151
152 VMOV V2.D[0], R0
153 RET
154
155 aes33to64:
156 VLD1 (R4), [V1.B16, V2.B16, V3.B16]
157 AESE V30.B16, V1.B16
158 AESMC V1.B16, V1.B16
159 AESE V30.B16, V2.B16
160 AESMC V2.B16, V2.B16
161 AESE V30.B16, V3.B16
162 AESMC V3.B16, V3.B16
163 SUB $32, R2, R10
164
165 VLD1.P (R0)(R10), [V4.B16, V5.B16]
166 VLD1 (R0), [V6.B16, V7.B16]
167
168 AESE V0.B16, V4.B16
169 AESMC V4.B16, V4.B16
170 AESE V1.B16, V5.B16
171 AESMC V5.B16, V5.B16
172 AESE V2.B16, V6.B16
173 AESMC V6.B16, V6.B16
174 AESE V3.B16, V7.B16
175 AESMC V7.B16, V7.B16
176
177 AESE V0.B16, V4.B16
178 AESMC V4.B16, V4.B16
179 AESE V1.B16, V5.B16
180 AESMC V5.B16, V5.B16
181 AESE V2.B16, V6.B16
182 AESMC V6.B16, V6.B16
183 AESE V3.B16, V7.B16
184 AESMC V7.B16, V7.B16
185
186 AESE V0.B16, V4.B16
187 AESE V1.B16, V5.B16
188 AESE V2.B16, V6.B16
189 AESE V3.B16, V7.B16
190
191 VEOR V6.B16, V4.B16, V4.B16
192 VEOR V7.B16, V5.B16, V5.B16
193 VEOR V5.B16, V4.B16, V4.B16
194
195 VMOV V4.D[0], R0
196 RET
197
198 aes65to128:
199 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
200 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
201 AESE V30.B16, V1.B16
202 AESMC V1.B16, V1.B16
203 AESE V30.B16, V2.B16
204 AESMC V2.B16, V2.B16
205 AESE V30.B16, V3.B16
206 AESMC V3.B16, V3.B16
207 AESE V30.B16, V4.B16
208 AESMC V4.B16, V4.B16
209 AESE V30.B16, V5.B16
210 AESMC V5.B16, V5.B16
211 AESE V30.B16, V6.B16
212 AESMC V6.B16, V6.B16
213 AESE V30.B16, V7.B16
214 AESMC V7.B16, V7.B16
215
216 SUB $64, R2, R10
217 VLD1.P (R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
218 VLD1 (R0), [V12.B16, V13.B16, V14.B16, V15.B16]
219 AESE V0.B16, V8.B16
220 AESMC V8.B16, V8.B16
221 AESE V1.B16, V9.B16
222 AESMC V9.B16, V9.B16
223 AESE V2.B16, V10.B16
224 AESMC V10.B16, V10.B16
225 AESE V3.B16, V11.B16
226 AESMC V11.B16, V11.B16
227 AESE V4.B16, V12.B16
228 AESMC V12.B16, V12.B16
229 AESE V5.B16, V13.B16
230 AESMC V13.B16, V13.B16
231 AESE V6.B16, V14.B16
232 AESMC V14.B16, V14.B16
233 AESE V7.B16, V15.B16
234 AESMC V15.B16, V15.B16
235
236 AESE V0.B16, V8.B16
237 AESMC V8.B16, V8.B16
238 AESE V1.B16, V9.B16
239 AESMC V9.B16, V9.B16
240 AESE V2.B16, V10.B16
241 AESMC V10.B16, V10.B16
242 AESE V3.B16, V11.B16
243 AESMC V11.B16, V11.B16
244 AESE V4.B16, V12.B16
245 AESMC V12.B16, V12.B16
246 AESE V5.B16, V13.B16
247 AESMC V13.B16, V13.B16
248 AESE V6.B16, V14.B16
249 AESMC V14.B16, V14.B16
250 AESE V7.B16, V15.B16
251 AESMC V15.B16, V15.B16
252
253 AESE V0.B16, V8.B16
254 AESE V1.B16, V9.B16
255 AESE V2.B16, V10.B16
256 AESE V3.B16, V11.B16
257 AESE V4.B16, V12.B16
258 AESE V5.B16, V13.B16
259 AESE V6.B16, V14.B16
260 AESE V7.B16, V15.B16
261
262 VEOR V12.B16, V8.B16, V8.B16
263 VEOR V13.B16, V9.B16, V9.B16
264 VEOR V14.B16, V10.B16, V10.B16
265 VEOR V15.B16, V11.B16, V11.B16
266 VEOR V10.B16, V8.B16, V8.B16
267 VEOR V11.B16, V9.B16, V9.B16
268 VEOR V9.B16, V8.B16, V8.B16
269
270 VMOV V8.D[0], R0
271 RET
272
273 aes129plus:
274 PRFM (R0), PLDL1KEEP
275 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
276 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
277 AESE V30.B16, V1.B16
278 AESMC V1.B16, V1.B16
279 AESE V30.B16, V2.B16
280 AESMC V2.B16, V2.B16
281 AESE V30.B16, V3.B16
282 AESMC V3.B16, V3.B16
283 AESE V30.B16, V4.B16
284 AESMC V4.B16, V4.B16
285 AESE V30.B16, V5.B16
286 AESMC V5.B16, V5.B16
287 AESE V30.B16, V6.B16
288 AESMC V6.B16, V6.B16
289 AESE V30.B16, V7.B16
290 AESMC V7.B16, V7.B16
291 ADD R0, R2, R10
292 SUB $128, R10, R10
293 VLD1.P 64(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
294 VLD1 (R10), [V12.B16, V13.B16, V14.B16, V15.B16]
295 SUB $1, R2, R2
296 LSR $7, R2, R2
297
298 aesloop:
299 AESE V8.B16, V0.B16
300 AESMC V0.B16, V0.B16
301 AESE V9.B16, V1.B16
302 AESMC V1.B16, V1.B16
303 AESE V10.B16, V2.B16
304 AESMC V2.B16, V2.B16
305 AESE V11.B16, V3.B16
306 AESMC V3.B16, V3.B16
307 AESE V12.B16, V4.B16
308 AESMC V4.B16, V4.B16
309 AESE V13.B16, V5.B16
310 AESMC V5.B16, V5.B16
311 AESE V14.B16, V6.B16
312 AESMC V6.B16, V6.B16
313 AESE V15.B16, V7.B16
314 AESMC V7.B16, V7.B16
315
316 VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16]
317 AESE V8.B16, V0.B16
318 AESMC V0.B16, V0.B16
319 AESE V9.B16, V1.B16
320 AESMC V1.B16, V1.B16
321 AESE V10.B16, V2.B16
322 AESMC V2.B16, V2.B16
323 AESE V11.B16, V3.B16
324 AESMC V3.B16, V3.B16
325
326 VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
327 AESE V12.B16, V4.B16
328 AESMC V4.B16, V4.B16
329 AESE V13.B16, V5.B16
330 AESMC V5.B16, V5.B16
331 AESE V14.B16, V6.B16
332 AESMC V6.B16, V6.B16
333 AESE V15.B16, V7.B16
334 AESMC V7.B16, V7.B16
335 SUB $1, R2, R2
336 CBNZ R2, aesloop
337
338 AESE V8.B16, V0.B16
339 AESMC V0.B16, V0.B16
340 AESE V9.B16, V1.B16
341 AESMC V1.B16, V1.B16
342 AESE V10.B16, V2.B16
343 AESMC V2.B16, V2.B16
344 AESE V11.B16, V3.B16
345 AESMC V3.B16, V3.B16
346 AESE V12.B16, V4.B16
347 AESMC V4.B16, V4.B16
348 AESE V13.B16, V5.B16
349 AESMC V5.B16, V5.B16
350 AESE V14.B16, V6.B16
351 AESMC V6.B16, V6.B16
352 AESE V15.B16, V7.B16
353 AESMC V7.B16, V7.B16
354
355 AESE V8.B16, V0.B16
356 AESMC V0.B16, V0.B16
357 AESE V9.B16, V1.B16
358 AESMC V1.B16, V1.B16
359 AESE V10.B16, V2.B16
360 AESMC V2.B16, V2.B16
361 AESE V11.B16, V3.B16
362 AESMC V3.B16, V3.B16
363 AESE V12.B16, V4.B16
364 AESMC V4.B16, V4.B16
365 AESE V13.B16, V5.B16
366 AESMC V5.B16, V5.B16
367 AESE V14.B16, V6.B16
368 AESMC V6.B16, V6.B16
369 AESE V15.B16, V7.B16
370 AESMC V7.B16, V7.B16
371
372 AESE V8.B16, V0.B16
373 AESE V9.B16, V1.B16
374 AESE V10.B16, V2.B16
375 AESE V11.B16, V3.B16
376 AESE V12.B16, V4.B16
377 AESE V13.B16, V5.B16
378 AESE V14.B16, V6.B16
379 AESE V15.B16, V7.B16
380
381 VEOR V0.B16, V1.B16, V0.B16
382 VEOR V2.B16, V3.B16, V2.B16
383 VEOR V4.B16, V5.B16, V4.B16
384 VEOR V6.B16, V7.B16, V6.B16
385 VEOR V0.B16, V2.B16, V0.B16
386 VEOR V4.B16, V6.B16, V4.B16
387 VEOR V4.B16, V0.B16, V0.B16
388
389 VMOV V0.D[0], R0
390 RET
391
View as plain text