summaryrefslogtreecommitdiff
path: root/string
diff options
context:
space:
mode:
authorAndrew Turner <andrew@FreeBSD.org>2024-02-20 09:02:15 +0000
committerAndrew Turner <andrew@FreeBSD.org>2024-02-20 09:02:15 +0000
commitedc5c0de794f521eb620d2b6cbaee2434442a8f3 (patch)
tree64dfc547c0b6398e9cf94bd8175b21db8a74c814 /string
parent29866ecb89620f1c798b7f5ff6710255f13aa52e (diff)
Update the Arm Optimized Routinesvendor/arm-optimized-routines/v24.01
Import the v24.01 release of the Arm Optimized Routines [1]. [1] https://github.com/ARM-software/optimized-routines/tree/v24.01 Sponsored by: Arm Ltd
Diffstat (limited to 'string')
-rw-r--r--string/aarch64/asmdefs.h14
-rw-r--r--string/aarch64/memcpy-advsimd.S62
-rw-r--r--string/aarch64/memcpy-mops.S21
-rw-r--r--string/aarch64/memmove-mops.S21
-rw-r--r--string/aarch64/memset-mops.S20
-rw-r--r--string/bench/memcpy.c5
-rw-r--r--string/include/stringlib.h7
-rw-r--r--string/test/memcpy.c5
-rw-r--r--string/test/memmove.c5
-rw-r--r--string/test/memset.c5
10 files changed, 132 insertions, 33 deletions
diff --git a/string/aarch64/asmdefs.h b/string/aarch64/asmdefs.h
index 069b146f4a69..131b95e1fea9 100644
--- a/string/aarch64/asmdefs.h
+++ b/string/aarch64/asmdefs.h
@@ -21,6 +21,19 @@
#define FEATURE_1_PAC 2
/* Add a NT_GNU_PROPERTY_TYPE_0 note. */
+#ifdef __ILP32__
+#define GNU_PROPERTY(type, value) \
+ .section .note.gnu.property, "a"; \
+ .p2align 2; \
+ .word 4; \
+ .word 12; \
+ .word 5; \
+ .asciz "GNU"; \
+ .word type; \
+ .word 4; \
+ .word value; \
+ .text
+#else
#define GNU_PROPERTY(type, value) \
.section .note.gnu.property, "a"; \
.p2align 3; \
@@ -33,6 +46,7 @@
.word value; \
.word 0; \
.text
+#endif
/* If set then the GNU Property Note section will be added to
mark objects to support BTI and PAC-RET. */
diff --git a/string/aarch64/memcpy-advsimd.S b/string/aarch64/memcpy-advsimd.S
index e6527d0dac2c..9d3027d4d3cd 100644
--- a/string/aarch64/memcpy-advsimd.S
+++ b/string/aarch64/memcpy-advsimd.S
@@ -1,7 +1,7 @@
/*
* memcpy - copy memory area
*
- * Copyright (c) 2019-2022, Arm Limited.
+ * Copyright (c) 2019-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -56,11 +56,12 @@ ENTRY (__memcpy_aarch64_simd)
PTR_ARG (1)
SIZE_ARG (2)
add srcend, src, count
- add dstend, dstin, count
cmp count, 128
b.hi L(copy_long)
+ add dstend, dstin, count
cmp count, 32
b.hi L(copy32_128)
+ nop
/* Small copies: 0..32 bytes. */
cmp count, 16
@@ -71,6 +72,18 @@ ENTRY (__memcpy_aarch64_simd)
str B_q, [dstend, -16]
ret
+ .p2align 4
+ /* Medium copies: 33..128 bytes. */
+L(copy32_128):
+ ldp A_q, B_q, [src]
+ ldp C_q, D_q, [srcend, -32]
+ cmp count, 64
+ b.hi L(copy128)
+ stp A_q, B_q, [dstin]
+ stp C_q, D_q, [dstend, -32]
+ ret
+
+ .p2align 4
/* Copy 8-15 bytes. */
L(copy16):
tbz count, 3, L(copy8)
@@ -80,7 +93,6 @@ L(copy16):
str A_h, [dstend, -8]
ret
- .p2align 3
/* Copy 4-7 bytes. */
L(copy8):
tbz count, 2, L(copy4)
@@ -90,31 +102,6 @@ L(copy8):
str B_lw, [dstend, -4]
ret
- /* Copy 0..3 bytes using a branchless sequence. */
-L(copy4):
- cbz count, L(copy0)
- lsr tmp1, count, 1
- ldrb A_lw, [src]
- ldrb C_lw, [srcend, -1]
- ldrb B_lw, [src, tmp1]
- strb A_lw, [dstin]
- strb B_lw, [dstin, tmp1]
- strb C_lw, [dstend, -1]
-L(copy0):
- ret
-
- .p2align 4
- /* Medium copies: 33..128 bytes. */
-L(copy32_128):
- ldp A_q, B_q, [src]
- ldp C_q, D_q, [srcend, -32]
- cmp count, 64
- b.hi L(copy128)
- stp A_q, B_q, [dstin]
- stp C_q, D_q, [dstend, -32]
- ret
-
- .p2align 4
/* Copy 65..128 bytes. */
L(copy128):
ldp E_q, F_q, [src, 32]
@@ -128,8 +115,24 @@ L(copy96):
stp C_q, D_q, [dstend, -32]
ret
+ /* Copy 0..3 bytes using a branchless sequence. */
+L(copy4):
+ cbz count, L(copy0)
+ lsr tmp1, count, 1
+ ldrb A_lw, [src]
+ ldrb C_lw, [srcend, -1]
+ ldrb B_lw, [src, tmp1]
+ strb A_lw, [dstin]
+ strb B_lw, [dstin, tmp1]
+ strb C_lw, [dstend, -1]
+L(copy0):
+ ret
+
+ .p2align 3
/* Copy more than 128 bytes. */
L(copy_long):
+ add dstend, dstin, count
+
/* Use backwards copy if there is an overlap. */
sub tmp1, dstin, src
cmp tmp1, count
@@ -166,6 +169,9 @@ L(copy64_from_end):
stp A_q, B_q, [dstend, -32]
ret
+ .p2align 4
+ nop
+
/* Large backwards copy for overlapping copies.
Copy 16 bytes and then align srcend to 16-byte alignment. */
L(copy_long_backwards):
diff --git a/string/aarch64/memcpy-mops.S b/string/aarch64/memcpy-mops.S
new file mode 100644
index 000000000000..b45c31418717
--- /dev/null
+++ b/string/aarch64/memcpy-mops.S
@@ -0,0 +1,21 @@
+/*
+ * memcpy using MOPS extension.
+ *
+ * Copyright (c) 2023, Arm Limited.
+ * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+ */
+
+#include "asmdefs.h"
+
+ENTRY (__memcpy_aarch64_mops)
+ PTR_ARG (0)
+ PTR_ARG (1)
+ SIZE_ARG (2)
+
+ mov x3, x0
+ .inst 0x19010443 /* cpyfp [x3]!, [x1]!, x2! */
+ .inst 0x19410443 /* cpyfm [x3]!, [x1]!, x2! */
+ .inst 0x19810443 /* cpyfe [x3]!, [x1]!, x2! */
+ ret
+
+END (__memcpy_aarch64_mops)
diff --git a/string/aarch64/memmove-mops.S b/string/aarch64/memmove-mops.S
new file mode 100644
index 000000000000..6c73017bb16f
--- /dev/null
+++ b/string/aarch64/memmove-mops.S
@@ -0,0 +1,21 @@
+/*
+ * memmove using MOPS extension.
+ *
+ * Copyright (c) 2023, Arm Limited.
+ * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+ */
+
+#include "asmdefs.h"
+
+ENTRY (__memmove_aarch64_mops)
+ PTR_ARG (0)
+ PTR_ARG (1)
+ SIZE_ARG (2)
+
+ mov x3, x0
+ .inst 0x1d010443 /* cpyp [x3]!, [x1]!, x2! */
+ .inst 0x1d410443 /* cpym [x3]!, [x1]!, x2! */
+ .inst 0x1d810443 /* cpye [x3]!, [x1]!, x2! */
+ ret
+
+END (__memmove_aarch64_mops)
diff --git a/string/aarch64/memset-mops.S b/string/aarch64/memset-mops.S
new file mode 100644
index 000000000000..ec791493bae9
--- /dev/null
+++ b/string/aarch64/memset-mops.S
@@ -0,0 +1,20 @@
+/*
+ * memset using MOPS extension.
+ *
+ * Copyright (c) 2023, Arm Limited.
+ * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+ */
+
+#include "asmdefs.h"
+
+ENTRY (__memset_aarch64_mops)
+ PTR_ARG (0)
+ SIZE_ARG (2)
+
+ mov x3, x0
+ .inst 0x19c10443 /* setp [x3]!, x2!, x1 */
+ .inst 0x19c14443 /* setm [x3]!, x2!, x1 */
+ .inst 0x19c18443 /* sete [x3]!, x2!, x1 */
+ ret
+
+END (__memset_aarch64_mops)
diff --git a/string/bench/memcpy.c b/string/bench/memcpy.c
index 1468663e51cd..b628f9b60d96 100644
--- a/string/bench/memcpy.c
+++ b/string/bench/memcpy.c
@@ -1,7 +1,7 @@
/*
* memcpy benchmark.
*
- * Copyright (c) 2020-2022, Arm Limited.
+ * Copyright (c) 2020-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -39,6 +39,9 @@ static const struct fun
# if __ARM_FEATURE_SVE
F(__memcpy_aarch64_sve)
# endif
+# if WANT_MOPS
+ F(__memcpy_aarch64_mops)
+# endif
#elif __arm__
F(__memcpy_arm)
#endif
diff --git a/string/include/stringlib.h b/string/include/stringlib.h
index f41a46446888..01da7ebfc18d 100644
--- a/string/include/stringlib.h
+++ b/string/include/stringlib.h
@@ -1,7 +1,7 @@
/*
* Public API.
*
- * Copyright (c) 2019-2022, Arm Limited.
+ * Copyright (c) 2019-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -52,6 +52,11 @@ size_t __strlen_aarch64_sve (const char *);
size_t __strnlen_aarch64_sve (const char *, size_t);
int __strncmp_aarch64_sve (const char *, const char *, size_t);
# endif
+# if WANT_MOPS
+void *__memcpy_aarch64_mops (void *__restrict, const void *__restrict, size_t);
+void *__memmove_aarch64_mops (void *__restrict, const void *__restrict, size_t);
+void *__memset_aarch64_mops (void *, int, size_t);
+# endif
# if __ARM_FEATURE_MEMORY_TAGGING
void *__mtag_tag_region (void *, size_t);
void *__mtag_tag_zero_region (void *, size_t);
diff --git a/string/test/memcpy.c b/string/test/memcpy.c
index fa15a95b2bda..dc95844bd45a 100644
--- a/string/test/memcpy.c
+++ b/string/test/memcpy.c
@@ -1,7 +1,7 @@
/*
* memcpy test.
*
- * Copyright (c) 2019-2022, Arm Limited.
+ * Copyright (c) 2019-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -31,6 +31,9 @@ static const struct fun
# if __ARM_FEATURE_SVE
F(__memcpy_aarch64_sve, 1)
# endif
+# if WANT_MOPS
+ F(__memcpy_aarch64_mops, 1)
+# endif
#elif __arm__
F(__memcpy_arm, 0)
#endif
diff --git a/string/test/memmove.c b/string/test/memmove.c
index 5d509c03affa..b85dd1e864ef 100644
--- a/string/test/memmove.c
+++ b/string/test/memmove.c
@@ -1,7 +1,7 @@
/*
* memmove test.
*
- * Copyright (c) 2019-2022, Arm Limited.
+ * Copyright (c) 2019-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -31,6 +31,9 @@ static const struct fun
# if __ARM_FEATURE_SVE
F(__memmove_aarch64_sve, 1)
# endif
+# if WANT_MOPS
+ F(__memmove_aarch64_mops, 1)
+# endif
#endif
{0, 0, 0}
// clang-format on
diff --git a/string/test/memset.c b/string/test/memset.c
index 5543f44bb026..7d09c267ffec 100644
--- a/string/test/memset.c
+++ b/string/test/memset.c
@@ -1,7 +1,7 @@
/*
* memset test.
*
- * Copyright (c) 2019-2020, Arm Limited.
+ * Copyright (c) 2019-2023, Arm Limited.
* SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
*/
@@ -25,6 +25,9 @@ static const struct fun
F(memset, 0)
#if __aarch64__
F(__memset_aarch64, 1)
+# if WANT_MOPS
+ F(__memset_aarch64_mops, 1)
+# endif
#elif __arm__
F(__memset_arm, 0)
#endif
generated by cgit v1.3 (git 2.53.0) at 2026-09-09 08:39:06 +0000