core/stdarch/crates/core_arch/src/x86_64/
amx.rs

1use crate::core_arch::{simd::*, x86::*};
2
3#[cfg(test)]
4use stdarch_test::assert_instr;
5
6/// Load tile configuration from a 64-byte memory location specified by mem_addr.
7/// The tile configuration format is specified below, and includes the tile type pallette,
8/// the number of bytes per row, and the number of rows. If the specified pallette_id is zero,
9/// that signifies the init state for both the tile config and the tile data, and the tiles are zeroed.
10/// Any invalid configurations will result in #GP fault.
11///
12/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_loadconfig&ig_expand=6875)
13#[inline]
14#[target_feature(enable = "amx-tile")]
15#[cfg_attr(test, assert_instr(ldtilecfg))]
16#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
17pub unsafe fn _tile_loadconfig(mem_addr: *const u8) {
18    ldtilecfg(mem_addr);
19}
20
21/// Stores the current tile configuration to a 64-byte memory location specified by mem_addr.
22/// The tile configuration format is specified below, and includes the tile type pallette,
23/// the number of bytes per row, and the number of rows. If tiles are not configured, all zeroes will be stored to memory.
24///
25/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_storeconfig&ig_expand=6879)
26#[inline]
27#[target_feature(enable = "amx-tile")]
28#[cfg_attr(test, assert_instr(sttilecfg))]
29#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
30pub unsafe fn _tile_storeconfig(mem_addr: *mut u8) {
31    sttilecfg(mem_addr);
32}
33
34/// Load tile rows from memory specifieid by base address and stride into destination tile dst using the tile configuration previously configured via _tile_loadconfig.
35///
36/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_loadd&ig_expand=6877)
37#[inline]
38#[rustc_legacy_const_generics(0)]
39#[target_feature(enable = "amx-tile")]
40#[cfg_attr(test, assert_instr(tileloadd, DST = 0))]
41#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
42pub unsafe fn _tile_loadd<const DST: i32>(base: *const u8, stride: usize) {
43    static_assert_uimm_bits!(DST, 3);
44    tileloadd64(DST as i8, base, stride);
45}
46
47/// Release the tile configuration to return to the init state, which releases all storage it currently holds.
48///
49/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_release&ig_expand=6878)
50#[inline]
51#[target_feature(enable = "amx-tile")]
52#[cfg_attr(test, assert_instr(tilerelease))]
53#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
54pub unsafe fn _tile_release() {
55    tilerelease();
56}
57
58/// Store the tile specified by src to memory specifieid by base address and stride using the tile configuration previously configured via _tile_loadconfig.
59///
60/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_stored&ig_expand=6881)
61#[inline]
62#[rustc_legacy_const_generics(0)]
63#[target_feature(enable = "amx-tile")]
64#[cfg_attr(test, assert_instr(tilestored, DST = 0))]
65#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
66pub unsafe fn _tile_stored<const DST: i32>(base: *mut u8, stride: usize) {
67    static_assert_uimm_bits!(DST, 3);
68    tilestored64(DST as i8, base, stride);
69}
70
71/// Load tile rows from memory specifieid by base address and stride into destination tile dst using the tile configuration
72/// previously configured via _tile_loadconfig. This intrinsic provides a hint to the implementation that the data will
73/// likely not be reused in the near future and the data caching can be optimized accordingly.
74///
75/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_stream_loadd&ig_expand=6883)
76#[inline]
77#[rustc_legacy_const_generics(0)]
78#[target_feature(enable = "amx-tile")]
79#[cfg_attr(test, assert_instr(tileloaddt1, DST = 0))]
80#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
81pub unsafe fn _tile_stream_loadd<const DST: i32>(base: *const u8, stride: usize) {
82    static_assert_uimm_bits!(DST, 3);
83    tileloaddt164(DST as i8, base, stride);
84}
85
86/// Zero the tile specified by tdest.
87///
88/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_zero&ig_expand=6885)
89#[inline]
90#[rustc_legacy_const_generics(0)]
91#[target_feature(enable = "amx-tile")]
92#[cfg_attr(test, assert_instr(tilezero, DST = 0))]
93#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
94pub unsafe fn _tile_zero<const DST: i32>() {
95    static_assert_uimm_bits!(DST, 3);
96    tilezero(DST as i8);
97}
98
99/// Compute dot-product of BF16 (16-bit) floating-point pairs in tiles a and b,
100/// accumulating the intermediate single-precision (32-bit) floating-point elements
101/// with elements in dst, and store the 32-bit result back to tile dst.
102///
103/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbf16ps&ig_expand=6864)
104#[inline]
105#[rustc_legacy_const_generics(0, 1, 2)]
106#[target_feature(enable = "amx-bf16")]
107#[cfg_attr(test, assert_instr(tdpbf16ps, DST = 0, A = 1, B = 2))]
108#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
109pub unsafe fn _tile_dpbf16ps<const DST: i32, const A: i32, const B: i32>() {
110    static_assert_uimm_bits!(DST, 3);
111    static_assert_uimm_bits!(A, 3);
112    static_assert_uimm_bits!(B, 3);
113    tdpbf16ps(DST as i8, A as i8, B as i8);
114}
115
116/// Compute dot-product of bytes in tiles with a source/destination accumulator.
117/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
118/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
119/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
120///
121/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbssd&ig_expand=6866)
122#[inline]
123#[rustc_legacy_const_generics(0, 1, 2)]
124#[target_feature(enable = "amx-int8")]
125#[cfg_attr(test, assert_instr(tdpbssd, DST = 0, A = 1, B = 2))]
126#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
127pub unsafe fn _tile_dpbssd<const DST: i32, const A: i32, const B: i32>() {
128    static_assert_uimm_bits!(DST, 3);
129    static_assert_uimm_bits!(A, 3);
130    static_assert_uimm_bits!(B, 3);
131    tdpbssd(DST as i8, A as i8, B as i8);
132}
133
134/// Compute dot-product of bytes in tiles with a source/destination accumulator.
135/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
136/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
137/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
138///
139/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbsud&ig_expand=6868)
140#[inline]
141#[rustc_legacy_const_generics(0, 1, 2)]
142#[target_feature(enable = "amx-int8")]
143#[cfg_attr(test, assert_instr(tdpbsud, DST = 0, A = 1, B = 2))]
144#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
145pub unsafe fn _tile_dpbsud<const DST: i32, const A: i32, const B: i32>() {
146    static_assert_uimm_bits!(DST, 3);
147    static_assert_uimm_bits!(A, 3);
148    static_assert_uimm_bits!(B, 3);
149    tdpbsud(DST as i8, A as i8, B as i8);
150}
151
152/// Compute dot-product of bytes in tiles with a source/destination accumulator.
153/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
154/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
155/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
156///
157/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbusd&ig_expand=6870)
158#[inline]
159#[rustc_legacy_const_generics(0, 1, 2)]
160#[target_feature(enable = "amx-int8")]
161#[cfg_attr(test, assert_instr(tdpbusd, DST = 0, A = 1, B = 2))]
162#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
163pub unsafe fn _tile_dpbusd<const DST: i32, const A: i32, const B: i32>() {
164    static_assert_uimm_bits!(DST, 3);
165    static_assert_uimm_bits!(A, 3);
166    static_assert_uimm_bits!(B, 3);
167    tdpbusd(DST as i8, A as i8, B as i8);
168}
169
170/// Compute dot-product of bytes in tiles with a source/destination accumulator.
171/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
172/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
173/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
174///
175/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbuud&ig_expand=6872)
176#[inline]
177#[rustc_legacy_const_generics(0, 1, 2)]
178#[target_feature(enable = "amx-int8")]
179#[cfg_attr(test, assert_instr(tdpbuud, DST = 0, A = 1, B = 2))]
180#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
181pub unsafe fn _tile_dpbuud<const DST: i32, const A: i32, const B: i32>() {
182    static_assert_uimm_bits!(DST, 3);
183    static_assert_uimm_bits!(A, 3);
184    static_assert_uimm_bits!(B, 3);
185    tdpbuud(DST as i8, A as i8, B as i8);
186}
187
188/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles a and b,
189/// accumulating the intermediate single-precision (32-bit) floating-point elements
190///  with elements in dst, and store the 32-bit result back to tile dst.
191///
192/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpfp16ps&ig_expand=6874)
193#[inline]
194#[rustc_legacy_const_generics(0, 1, 2)]
195#[target_feature(enable = "amx-fp16")]
196#[cfg_attr(test, assert_instr(tdpfp16ps, DST = 0, A = 1, B = 2))]
197#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
198pub unsafe fn _tile_dpfp16ps<const DST: i32, const A: i32, const B: i32>() {
199    static_assert_uimm_bits!(DST, 3);
200    static_assert_uimm_bits!(A, 3);
201    static_assert_uimm_bits!(B, 3);
202    tdpfp16ps(DST as i8, A as i8, B as i8);
203}
204
205/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
206/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
207/// Calculates the imaginary part of the result. For each possible combination of (row of a, column of b),
208/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
209/// The imaginary part of the a element is multiplied with the real part of the corresponding b element, and the real part of
210/// the a element is multiplied with the imaginary part of the corresponding b elements. The two accumulated results are added,
211/// and then accumulated into the corresponding row and column of dst.
212///
213/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_cmmimfp16ps&ig_expand=6860)
214#[inline]
215#[rustc_legacy_const_generics(0, 1, 2)]
216#[target_feature(enable = "amx-complex")]
217#[cfg_attr(test, assert_instr(tcmmimfp16ps, DST = 0, A = 1, B = 2))]
218#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
219pub unsafe fn _tile_cmmimfp16ps<const DST: i32, const A: i32, const B: i32>() {
220    static_assert_uimm_bits!(DST, 3);
221    static_assert_uimm_bits!(A, 3);
222    static_assert_uimm_bits!(B, 3);
223    tcmmimfp16ps(DST as i8, A as i8, B as i8);
224}
225
226/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
227/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
228/// Calculates the real part of the result. For each possible combination of (row of a, column of b),
229/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
230/// The real part of the a element is multiplied with the real part of the corresponding b element, and the negated imaginary part of
231/// the a element is multiplied with the imaginary part of the corresponding b elements.
232/// The two accumulated results are added, and then accumulated into the corresponding row and column of dst.
233///
234/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_cmmrlfp16ps&ig_expand=6862)
235#[inline]
236#[rustc_legacy_const_generics(0, 1, 2)]
237#[target_feature(enable = "amx-complex")]
238#[cfg_attr(test, assert_instr(tcmmrlfp16ps, DST = 0, A = 1, B = 2))]
239#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
240pub unsafe fn _tile_cmmrlfp16ps<const DST: i32, const A: i32, const B: i32>() {
241    static_assert_uimm_bits!(DST, 3);
242    static_assert_uimm_bits!(A, 3);
243    static_assert_uimm_bits!(B, 3);
244    tcmmrlfp16ps(DST as i8, A as i8, B as i8);
245}
246
247/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and BF8 (8-bit E5M2)
248/// floating-point elements in tile b, accumulating the intermediate single-precision
249/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
250/// back to tile dst.
251#[inline]
252#[rustc_legacy_const_generics(0, 1, 2)]
253#[target_feature(enable = "amx-fp8")]
254#[cfg_attr(
255    all(test, any(target_os = "linux", target_env = "msvc")),
256    assert_instr(tdpbf8ps, DST = 0, A = 1, B = 2)
257)]
258#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
259pub unsafe fn _tile_dpbf8ps<const DST: i32, const A: i32, const B: i32>() {
260    static_assert_uimm_bits!(DST, 3);
261    static_assert_uimm_bits!(A, 3);
262    static_assert_uimm_bits!(B, 3);
263    tdpbf8ps(DST as i8, A as i8, B as i8);
264}
265
266/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and HF8
267/// (8-bit E4M3) floating-point elements in tile b, accumulating the intermediate single-precision
268/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
269/// back to tile dst.
270#[inline]
271#[rustc_legacy_const_generics(0, 1, 2)]
272#[target_feature(enable = "amx-fp8")]
273#[cfg_attr(
274    all(test, any(target_os = "linux", target_env = "msvc")),
275    assert_instr(tdpbhf8ps, DST = 0, A = 1, B = 2)
276)]
277#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
278pub unsafe fn _tile_dpbhf8ps<const DST: i32, const A: i32, const B: i32>() {
279    static_assert_uimm_bits!(DST, 3);
280    static_assert_uimm_bits!(A, 3);
281    static_assert_uimm_bits!(B, 3);
282    tdpbhf8ps(DST as i8, A as i8, B as i8);
283}
284
285/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and BF8
286/// (8-bit E5M2) floating-point elements in tile b, accumulating the intermediate single-precision
287/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
288/// back to tile dst.
289#[inline]
290#[rustc_legacy_const_generics(0, 1, 2)]
291#[target_feature(enable = "amx-fp8")]
292#[cfg_attr(
293    all(test, any(target_os = "linux", target_env = "msvc")),
294    assert_instr(tdphbf8ps, DST = 0, A = 1, B = 2)
295)]
296#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
297pub unsafe fn _tile_dphbf8ps<const DST: i32, const A: i32, const B: i32>() {
298    static_assert_uimm_bits!(DST, 3);
299    static_assert_uimm_bits!(A, 3);
300    static_assert_uimm_bits!(B, 3);
301    tdphbf8ps(DST as i8, A as i8, B as i8);
302}
303
304/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and HF8 (8-bit E4M3)
305/// floating-point elements in tile b, accumulating the intermediate single-precision
306/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
307/// back to tile dst.
308#[inline]
309#[rustc_legacy_const_generics(0, 1, 2)]
310#[target_feature(enable = "amx-fp8")]
311#[cfg_attr(
312    all(test, any(target_os = "linux", target_env = "msvc")),
313    assert_instr(tdphf8ps, DST = 0, A = 1, B = 2)
314)]
315#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
316pub unsafe fn _tile_dphf8ps<const DST: i32, const A: i32, const B: i32>() {
317    static_assert_uimm_bits!(DST, 3);
318    static_assert_uimm_bits!(A, 3);
319    static_assert_uimm_bits!(B, 3);
320    tdphf8ps(DST as i8, A as i8, B as i8);
321}
322
323/// Load tile rows from memory specified by base address and stride into destination tile dst
324/// using the tile configuration previously configured via _tile_loadconfig.
325/// Additionally, this intrinsic indicates the source memory location is likely to become
326/// read-shared by multiple processors, i.e., read in the future by at least one other processor
327/// before it is written, assuming it is ever written in the future.
328#[inline]
329#[rustc_legacy_const_generics(0)]
330#[target_feature(enable = "amx-movrs")]
331#[cfg_attr(
332    all(test, any(target_os = "linux", target_env = "msvc")),
333    assert_instr(tileloaddrs, DST = 0)
334)]
335#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
336pub unsafe fn _tile_loaddrs<const DST: i32>(base: *const u8, stride: usize) {
337    static_assert_uimm_bits!(DST, 3);
338    tileloaddrs64(DST as i8, base, stride);
339}
340
341/// Load tile rows from memory specified by base address and stride into destination tile dst
342/// using the tile configuration previously configured via _tile_loadconfig.
343/// Provides a hint to the implementation that the data would be reused but does not need
344/// to be resident in the nearest cache levels.
345/// Additionally, this intrinsic indicates the source memory location is likely to become
346/// read-shared by multiple processors, i.e., read in the future by at least one other processor
347/// before it is written, assuming it is ever written in the future.
348#[inline]
349#[rustc_legacy_const_generics(0)]
350#[target_feature(enable = "amx-movrs")]
351#[cfg_attr(
352    all(test, any(target_os = "linux", target_env = "msvc")),
353    assert_instr(tileloaddrst1, DST = 0)
354)]
355#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
356pub unsafe fn _tile_stream_loaddrs<const DST: i32>(base: *const u8, stride: usize) {
357    static_assert_uimm_bits!(DST, 3);
358    tileloaddrst164(DST as i8, base, stride);
359}
360
361/// Perform matrix multiplication of two tiles a and b, containing packed single precision (32-bit)
362/// floating-point elements, which are converted to TF32 (tensor-float32) format, and accumulate the
363///  results into a packed single precision tile.
364/// For each possible combination of (row of a, column of b), it performs
365///  - convert to TF32
366///  - multiply the corresponding elements of a and b
367///  - accumulate the results into the corresponding row and column of dst using round-to-nearest-even
368/// rounding mode.
369/// Output FP32 denormals are always flushed to zero, input single precision denormals are always
370/// handled and *not* treated as zero.
371#[inline]
372#[rustc_legacy_const_generics(0, 1, 2)]
373#[target_feature(enable = "amx-tf32")]
374#[cfg_attr(
375    all(test, any(target_os = "linux", target_env = "msvc")),
376    assert_instr(tmmultf32ps, DST = 0, A = 1, B = 2)
377)]
378#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
379pub unsafe fn _tile_mmultf32ps<const DST: i32, const A: i32, const B: i32>() {
380    static_assert_uimm_bits!(DST, 3);
381    static_assert_uimm_bits!(A, 3);
382    static_assert_uimm_bits!(B, 3);
383    tmmultf32ps(DST as i8, A as i8, B as i8);
384}
385
386/// Moves a row from a tile register to a zmm register, converting the packed 32-bit signed integer
387/// elements to packed single-precision (32-bit) floating-point elements.
388#[inline]
389#[rustc_legacy_const_generics(0)]
390#[target_feature(enable = "amx-avx512,avx10.2")]
391#[cfg_attr(
392    all(test, any(target_os = "linux", target_env = "msvc")),
393    assert_instr(tcvtrowd2ps, TILE = 0)
394)]
395#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
396pub unsafe fn _tile_cvtrowd2ps<const TILE: i32>(row: u32) -> __m512 {
397    static_assert_uimm_bits!(TILE, 3);
398    tcvtrowd2ps(TILE as i8, row).as_m512()
399}
400
401/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
402/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
403/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
404#[inline]
405#[rustc_legacy_const_generics(0)]
406#[target_feature(enable = "amx-avx512,avx10.2")]
407#[cfg_attr(
408    all(test, any(target_os = "linux", target_env = "msvc")),
409    assert_instr(tcvtrowps2phh, TILE = 0)
410)]
411#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
412pub unsafe fn _tile_cvtrowps2phh<const TILE: i32>(row: u32) -> __m512h {
413    static_assert_uimm_bits!(TILE, 3);
414    tcvtrowps2phh(TILE as i8, row).as_m512h()
415}
416
417/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
418/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
419/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
420#[inline]
421#[rustc_legacy_const_generics(0)]
422#[target_feature(enable = "amx-avx512,avx10.2")]
423#[cfg_attr(
424    all(test, any(target_os = "linux", target_env = "msvc")),
425    assert_instr(tcvtrowps2phl, TILE = 0)
426)]
427#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
428pub unsafe fn _tile_cvtrowps2phl<const TILE: i32>(row: u32) -> __m512h {
429    static_assert_uimm_bits!(TILE, 3);
430    tcvtrowps2phl(TILE as i8, row).as_m512h()
431}
432
433/// Moves one row of tile data into a zmm vector register
434#[inline]
435#[rustc_legacy_const_generics(0)]
436#[target_feature(enable = "amx-avx512,avx10.2")]
437#[cfg_attr(
438    all(test, any(target_os = "linux", target_env = "msvc")),
439    assert_instr(tilemovrow, TILE = 0)
440)]
441#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
442pub unsafe fn _tile_movrow<const TILE: i32>(row: u32) -> __m512i {
443    static_assert_uimm_bits!(TILE, 3);
444    tilemovrow(TILE as i8, row).as_m512i()
445}
446
447#[allow(improper_ctypes)]
448unsafe extern "C" {
449    #[link_name = "llvm.x86.ldtilecfg"]
450    fn ldtilecfg(mem_addr: *const u8);
451    #[link_name = "llvm.x86.sttilecfg"]
452    fn sttilecfg(mem_addr: *mut u8);
453    #[link_name = "llvm.x86.tileloadd64"]
454    fn tileloadd64(dst: i8, base: *const u8, stride: usize);
455    #[link_name = "llvm.x86.tileloaddt164"]
456    fn tileloaddt164(dst: i8, base: *const u8, stride: usize);
457    #[link_name = "llvm.x86.tilerelease"]
458    fn tilerelease();
459    #[link_name = "llvm.x86.tilestored64"]
460    fn tilestored64(dst: i8, base: *mut u8, stride: usize);
461    #[link_name = "llvm.x86.tilezero"]
462    fn tilezero(dst: i8);
463    #[link_name = "llvm.x86.tdpbf16ps"]
464    fn tdpbf16ps(dst: i8, a: i8, b: i8);
465    #[link_name = "llvm.x86.tdpbuud"]
466    fn tdpbuud(dst: i8, a: i8, b: i8);
467    #[link_name = "llvm.x86.tdpbusd"]
468    fn tdpbusd(dst: i8, a: i8, b: i8);
469    #[link_name = "llvm.x86.tdpbsud"]
470    fn tdpbsud(dst: i8, a: i8, b: i8);
471    #[link_name = "llvm.x86.tdpbssd"]
472    fn tdpbssd(dst: i8, a: i8, b: i8);
473    #[link_name = "llvm.x86.tdpfp16ps"]
474    fn tdpfp16ps(dst: i8, a: i8, b: i8);
475    #[link_name = "llvm.x86.tcmmimfp16ps"]
476    fn tcmmimfp16ps(dst: i8, a: i8, b: i8);
477    #[link_name = "llvm.x86.tcmmrlfp16ps"]
478    fn tcmmrlfp16ps(dst: i8, a: i8, b: i8);
479    #[link_name = "llvm.x86.tdpbf8ps"]
480    fn tdpbf8ps(dst: i8, a: i8, b: i8);
481    #[link_name = "llvm.x86.tdpbhf8ps"]
482    fn tdpbhf8ps(dst: i8, a: i8, b: i8);
483    #[link_name = "llvm.x86.tdphbf8ps"]
484    fn tdphbf8ps(dst: i8, a: i8, b: i8);
485    #[link_name = "llvm.x86.tdphf8ps"]
486    fn tdphf8ps(dst: i8, a: i8, b: i8);
487    #[link_name = "llvm.x86.tileloaddrs64"]
488    fn tileloaddrs64(dst: i8, base: *const u8, stride: usize);
489    #[link_name = "llvm.x86.tileloaddrst164"]
490    fn tileloaddrst164(dst: i8, base: *const u8, stride: usize);
491    #[link_name = "llvm.x86.tmmultf32ps"]
492    fn tmmultf32ps(dst: i8, a: i8, b: i8);
493    #[link_name = "llvm.x86.tcvtrowd2ps"]
494    fn tcvtrowd2ps(tile: i8, row: u32) -> f32x16;
495    #[link_name = "llvm.x86.tcvtrowps2phh"]
496    fn tcvtrowps2phh(tile: i8, row: u32) -> f16x32;
497    #[link_name = "llvm.x86.tcvtrowps2phl"]
498    fn tcvtrowps2phl(tile: i8, row: u32) -> f16x32;
499    #[link_name = "llvm.x86.tilemovrow"]
500    fn tilemovrow(tile: i8, row: u32) -> i32x16;
501}
502
503#[cfg(test)]
504mod tests {
505    use crate::core_arch::x86::_mm_cvtness_sbh;
506    use crate::core_arch::x86_64::*;
507    use core::{array, mem::transmute};
508    use stdarch_test::simd_test;
509    #[cfg(target_os = "linux")]
510    use syscalls::{Sysno, syscall};
511
512    #[allow(non_camel_case_types)]
513    #[repr(packed)]
514    #[derive(Copy, Clone, Default, Debug, PartialEq)]
515    struct __tilecfg {
516        /// 0 `or` 1
517        palette: u8,
518        start_row: u8,
519        /// reserved, must be zero
520        reserved_a0: [u8; 14],
521        /// number of bytes of one row in each tile
522        colsb: [u16; 8],
523        /// reserved, must be zero
524        reserved_b0: [u16; 8],
525        /// number of rows in each tile
526        rows: [u8; 8],
527        /// reserved, must be zero
528        reserved_c0: [u8; 8],
529    }
530
531    impl __tilecfg {
532        fn new(palette: u8, start_row: u8, colsb: [u16; 8], rows: [u8; 8]) -> Self {
533            Self {
534                palette,
535                start_row,
536                reserved_a0: [0u8; 14],
537                colsb,
538                reserved_b0: [0u16; 8],
539                rows,
540                reserved_c0: [0u8; 8],
541            }
542        }
543
544        const fn as_ptr(&self) -> *const u8 {
545            self as *const Self as *const u8
546        }
547
548        fn as_mut_ptr(&mut self) -> *mut u8 {
549            self as *mut Self as *mut u8
550        }
551    }
552
553    #[cfg(not(target_os = "linux"))]
554    #[target_feature(enable = "amx-tile")]
555    fn _init_amx() {}
556
557    #[cfg(target_os = "linux")]
558    #[target_feature(enable = "amx-tile")]
559    #[inline]
560    unsafe fn _init_amx() {
561        let mut ret: usize;
562        let mut xfeatures: usize = 0;
563        ret = syscall!(Sysno::arch_prctl, 0x1022, &mut xfeatures as *mut usize)
564            .expect("arch_prctl ARCH_GET_XCOMP_PERM syscall failed");
565        if ret != 0 {
566            panic!("Failed to get XFEATURES");
567        } else {
568            match 0b11 & (xfeatures >> 17) {
569                0 => panic!("AMX is not available"),
570                1 => {
571                    ret = syscall!(Sysno::arch_prctl, 0x1023, 18)
572                        .expect("arch_prctl ARCH_REQ_XCOMP_PERM syscall failed");
573                    if ret != 0 {
574                        panic!("Failed to enable AMX");
575                    }
576                }
577                3 => {}
578                _ => unreachable!(),
579            }
580        }
581    }
582
583    #[simd_test(enable = "amx-tile")]
584    unsafe fn test_tile_loadconfig() {
585        let config = __tilecfg::default();
586        _tile_loadconfig(config.as_ptr());
587        _tile_release();
588    }
589
590    #[simd_test(enable = "amx-tile")]
591    unsafe fn test_tile_storeconfig() {
592        let config = __tilecfg::new(1, 0, [32; 8], [8; 8]);
593        _tile_loadconfig(config.as_ptr());
594        let mut _config = __tilecfg::default();
595        _tile_storeconfig(_config.as_mut_ptr());
596        _tile_release();
597        assert_eq!(config, _config);
598    }
599
600    #[simd_test(enable = "amx-tile")]
601    unsafe fn test_tile_zero() {
602        _init_amx();
603        let mut config = __tilecfg::default();
604        config.palette = 1;
605        config.colsb[0] = 64;
606        config.rows[0] = 16;
607        _tile_loadconfig(config.as_ptr());
608        _tile_zero::<0>();
609        let mut out = [[1_i8; 64]; 16];
610        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
611        _tile_release();
612        assert_eq!(out, [[0; 64]; 16]);
613    }
614
615    #[simd_test(enable = "amx-tile")]
616    unsafe fn test_tile_stored() {
617        _init_amx();
618        let mut config = __tilecfg::default();
619        config.palette = 1;
620        config.colsb[0] = 64;
621        config.rows[0] = 16;
622        _tile_loadconfig(config.as_ptr());
623        _tile_zero::<0>();
624        let mut out = [[1_i8; 64]; 16];
625        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
626        _tile_release();
627        assert_eq!(out, [[0; 64]; 16]);
628    }
629
630    #[simd_test(enable = "amx-tile")]
631    unsafe fn test_tile_loadd() {
632        _init_amx();
633        let mut config = __tilecfg::default();
634        config.palette = 1;
635        config.colsb[0] = 64;
636        config.rows[0] = 16;
637        _tile_loadconfig(config.as_ptr());
638        _tile_zero::<0>();
639        let mat = [1_i8; 1024];
640        _tile_loadd::<0>(&mat as *const i8 as *const u8, 64);
641        let mut out = [[0_i8; 64]; 16];
642        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
643        _tile_release();
644        assert_eq!(out, [[1; 64]; 16]);
645    }
646
647    #[simd_test(enable = "amx-tile")]
648    unsafe fn test_tile_stream_loadd() {
649        _init_amx();
650        let mut config = __tilecfg::default();
651        config.palette = 1;
652        config.colsb[0] = 64;
653        config.rows[0] = 16;
654        _tile_loadconfig(config.as_ptr());
655        _tile_zero::<0>();
656        let mat = [1_i8; 1024];
657        _tile_stream_loadd::<0>(&mat as *const i8 as *const u8, 64);
658        let mut out = [[0_i8; 64]; 16];
659        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
660        _tile_release();
661        assert_eq!(out, [[1; 64]; 16]);
662    }
663
664    #[simd_test(enable = "amx-tile")]
665    unsafe fn test_tile_release() {
666        _tile_release();
667    }
668
669    #[simd_test(enable = "amx-bf16,avx512f")]
670    unsafe fn test_tile_dpbf16ps() {
671        _init_amx();
672        let bf16_1: u16 = _mm_cvtness_sbh(1.0).to_bits();
673        let bf16_2: u16 = _mm_cvtness_sbh(2.0).to_bits();
674        let ones: [u8; 1024] = transmute([bf16_1; 512]);
675        let twos: [u8; 1024] = transmute([bf16_2; 512]);
676        let mut res = [[0f32; 16]; 16];
677        let mut config = __tilecfg::default();
678        config.palette = 1;
679        (0..=2).for_each(|i| {
680            config.colsb[i] = 64;
681            config.rows[i] = 16;
682        });
683        _tile_loadconfig(config.as_ptr());
684        _tile_zero::<0>();
685        _tile_loadd::<1>(&ones as *const u8, 64);
686        _tile_loadd::<2>(&twos as *const u8, 64);
687        _tile_dpbf16ps::<0, 1, 2>();
688        _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64);
689        _tile_release();
690        assert_eq!(res, [[64f32; 16]; 16]);
691    }
692
693    #[simd_test(enable = "amx-int8")]
694    unsafe fn test_tile_dpbssd() {
695        _init_amx();
696        let ones = [-1_i8; 1024];
697        let twos = [-2_i8; 1024];
698        let mut res = [[0_i32; 16]; 16];
699        let mut config = __tilecfg::default();
700        config.palette = 1;
701        (0..=2).for_each(|i| {
702            config.colsb[i] = 64;
703            config.rows[i] = 16;
704        });
705        _tile_loadconfig(config.as_ptr());
706        _tile_zero::<0>();
707        _tile_loadd::<1>(&ones as *const i8 as *const u8, 64);
708        _tile_loadd::<2>(&twos as *const i8 as *const u8, 64);
709        _tile_dpbssd::<0, 1, 2>();
710        _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64);
711        _tile_release();
712        assert_eq!(res, [[128_i32; 16]; 16]);
713    }
714
715    #[simd_test(enable = "amx-int8")]
716    unsafe fn test_tile_dpbsud() {
717        _init_amx();
718        let ones = [-1_i8; 1024];
719        let twos = [2_u8; 1024];
720        let mut res = [[0_i32; 16]; 16];
721        let mut config = __tilecfg::default();
722        config.palette = 1;
723        (0..=2).for_each(|i| {
724            config.colsb[i] = 64;
725            config.rows[i] = 16;
726        });
727        _tile_loadconfig(config.as_ptr());
728        _tile_zero::<0>();
729        _tile_loadd::<1>(&ones as *const i8 as *const u8, 64);
730        _tile_loadd::<2>(&twos as *const u8, 64);
731        _tile_dpbsud::<0, 1, 2>();
732        _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64);
733        _tile_release();
734        assert_eq!(res, [[-128_i32; 16]; 16]);
735    }
736
737    #[simd_test(enable = "amx-int8")]
738    unsafe fn test_tile_dpbusd() {
739        _init_amx();
740        let ones = [1_u8; 1024];
741        let twos = [-2_i8; 1024];
742        let mut res = [[0_i32; 16]; 16];
743        let mut config = __tilecfg::default();
744        config.palette = 1;
745        (0..=2).for_each(|i| {
746            config.colsb[i] = 64;
747            config.rows[i] = 16;
748        });
749        _tile_loadconfig(config.as_ptr());
750        _tile_zero::<0>();
751        _tile_loadd::<1>(&ones as *const u8, 64);
752        _tile_loadd::<2>(&twos as *const i8 as *const u8, 64);
753        _tile_dpbusd::<0, 1, 2>();
754        _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64);
755        _tile_release();
756        assert_eq!(res, [[-128_i32; 16]; 16]);
757    }
758
759    #[simd_test(enable = "amx-int8")]
760    unsafe fn test_tile_dpbuud() {
761        _init_amx();
762        let ones = [1_u8; 1024];
763        let twos = [2_u8; 1024];
764        let mut res = [[0_i32; 16]; 16];
765        let mut config = __tilecfg::default();
766        config.palette = 1;
767        (0..=2).for_each(|i| {
768            config.colsb[i] = 64;
769            config.rows[i] = 16;
770        });
771        _tile_loadconfig(config.as_ptr());
772        _tile_zero::<0>();
773        _tile_loadd::<1>(&ones as *const u8, 64);
774        _tile_loadd::<2>(&twos as *const u8, 64);
775        _tile_dpbuud::<0, 1, 2>();
776        _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64);
777        _tile_release();
778        assert_eq!(res, [[128_i32; 16]; 16]);
779    }
780
781    #[simd_test(enable = "amx-fp16")]
782    unsafe fn test_tile_dpfp16ps() {
783        _init_amx();
784        let ones = [1f16; 512];
785        let twos = [2f16; 512];
786        let mut res = [[0f32; 16]; 16];
787        let mut config = __tilecfg::default();
788        config.palette = 1;
789        (0..=2).for_each(|i| {
790            config.colsb[i] = 64;
791            config.rows[i] = 16;
792        });
793        _tile_loadconfig(config.as_ptr());
794        _tile_zero::<0>();
795        _tile_loadd::<1>(&ones as *const f16 as *const u8, 64);
796        _tile_loadd::<2>(&twos as *const f16 as *const u8, 64);
797        _tile_dpfp16ps::<0, 1, 2>();
798        _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64);
799        _tile_release();
800        assert_eq!(res, [[64f32; 16]; 16]);
801    }
802
803    #[simd_test(enable = "amx-complex")]
804    unsafe fn test_tile_cmmimfp16ps() {
805        _init_amx();
806        let ones = [1f16; 512];
807        let twos = [2f16; 512];
808        let mut res = [[0f32; 16]; 16];
809        let mut config = __tilecfg::default();
810        config.palette = 1;
811        (0..=2).for_each(|i| {
812            config.colsb[i] = 64;
813            config.rows[i] = 16;
814        });
815        _tile_loadconfig(config.as_ptr());
816        _tile_zero::<0>();
817        _tile_loadd::<1>(&ones as *const f16 as *const u8, 64);
818        _tile_loadd::<2>(&twos as *const f16 as *const u8, 64);
819        _tile_cmmimfp16ps::<0, 1, 2>();
820        _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64);
821        _tile_release();
822        assert_eq!(res, [[64f32; 16]; 16]);
823    }
824
825    #[simd_test(enable = "amx-complex")]
826    unsafe fn test_tile_cmmrlfp16ps() {
827        _init_amx();
828        let ones = [1f16; 512];
829        let twos = [2f16; 512];
830        let mut res = [[0f32; 16]; 16];
831        let mut config = __tilecfg::default();
832        config.palette = 1;
833        (0..=2).for_each(|i| {
834            config.colsb[i] = 64;
835            config.rows[i] = 16;
836        });
837        _tile_loadconfig(config.as_ptr());
838        _tile_zero::<0>();
839        _tile_loadd::<1>(&ones as *const f16 as *const u8, 64);
840        _tile_loadd::<2>(&twos as *const f16 as *const u8, 64);
841        _tile_cmmrlfp16ps::<0, 1, 2>();
842        _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64);
843        _tile_release();
844        assert_eq!(res, [[0f32; 16]; 16]);
845    }
846
847    const BF8_ONE: u8 = 0x3c;
848    const BF8_TWO: u8 = 0x40;
849    const HF8_ONE: u8 = 0x38;
850    const HF8_TWO: u8 = 0x40;
851
852    #[simd_test(enable = "amx-fp8")]
853    unsafe fn test_tile_dpbf8ps() {
854        _init_amx();
855        let ones = [BF8_ONE; 1024];
856        let twos = [BF8_TWO; 1024];
857        let mut res = [[0.0_f32; 16]; 16];
858        let mut config = __tilecfg::default();
859        config.palette = 1;
860        (0..=2).for_each(|i| {
861            config.colsb[i] = 64;
862            config.rows[i] = 16;
863        });
864        _tile_loadconfig(config.as_ptr());
865        _tile_zero::<0>();
866        _tile_loadd::<1>(&ones as *const u8, 64);
867        _tile_loadd::<2>(&twos as *const u8, 64);
868        _tile_dpbf8ps::<0, 1, 2>();
869        _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
870        _tile_release();
871        assert_eq!(res, [[128.0_f32; 16]; 16]);
872    }
873
874    #[simd_test(enable = "amx-fp8")]
875    unsafe fn test_tile_dpbhf8ps() {
876        _init_amx();
877        let ones = [BF8_ONE; 1024];
878        let twos = [HF8_TWO; 1024];
879        let mut res = [[0.0_f32; 16]; 16];
880        let mut config = __tilecfg::default();
881        config.palette = 1;
882        (0..=2).for_each(|i| {
883            config.colsb[i] = 64;
884            config.rows[i] = 16;
885        });
886        _tile_loadconfig(config.as_ptr());
887        _tile_zero::<0>();
888        _tile_loadd::<1>(&ones as *const u8, 64);
889        _tile_loadd::<2>(&twos as *const u8, 64);
890        _tile_dpbhf8ps::<0, 1, 2>();
891        _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
892        _tile_release();
893        assert_eq!(res, [[128.0_f32; 16]; 16]);
894    }
895
896    #[simd_test(enable = "amx-fp8")]
897    unsafe fn test_tile_dphbf8ps() {
898        _init_amx();
899        let ones = [HF8_ONE; 1024];
900        let twos = [BF8_TWO; 1024];
901        let mut res = [[0.0_f32; 16]; 16];
902        let mut config = __tilecfg::default();
903        config.palette = 1;
904        (0..=2).for_each(|i| {
905            config.colsb[i] = 64;
906            config.rows[i] = 16;
907        });
908        _tile_loadconfig(config.as_ptr());
909        _tile_zero::<0>();
910        _tile_loadd::<1>(&ones as *const u8, 64);
911        _tile_loadd::<2>(&twos as *const u8, 64);
912        _tile_dphbf8ps::<0, 1, 2>();
913        _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
914        _tile_release();
915        assert_eq!(res, [[128.0_f32; 16]; 16]);
916    }
917
918    #[simd_test(enable = "amx-fp8")]
919    unsafe fn test_tile_dphf8ps() {
920        _init_amx();
921        let ones = [HF8_ONE; 1024];
922        let twos = [HF8_TWO; 1024];
923        let mut res = [[0.0_f32; 16]; 16];
924        let mut config = __tilecfg::default();
925        config.palette = 1;
926        (0..=2).for_each(|i| {
927            config.colsb[i] = 64;
928            config.rows[i] = 16;
929        });
930        _tile_loadconfig(config.as_ptr());
931        _tile_zero::<0>();
932        _tile_loadd::<1>(&ones as *const u8, 64);
933        _tile_loadd::<2>(&twos as *const u8, 64);
934        _tile_dphf8ps::<0, 1, 2>();
935        _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
936        _tile_release();
937        assert_eq!(res, [[128.0_f32; 16]; 16]);
938    }
939
940    #[simd_test(enable = "amx-movrs")]
941    unsafe fn test_tile_loaddrs() {
942        _init_amx();
943        let mut config = __tilecfg::default();
944        config.palette = 1;
945        config.colsb[0] = 64;
946        config.rows[0] = 16;
947        _tile_loadconfig(config.as_ptr());
948        _tile_zero::<0>();
949        let mat = [1_i8; 1024];
950        _tile_loaddrs::<0>(&mat as *const i8 as *const u8, 64);
951        let mut out = [[0_i8; 64]; 16];
952        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
953        _tile_release();
954        assert_eq!(out, [[1; 64]; 16]);
955    }
956
957    #[simd_test(enable = "amx-movrs")]
958    unsafe fn test_tile_stream_loaddrs() {
959        _init_amx();
960        let mut config = __tilecfg::default();
961        config.palette = 1;
962        config.colsb[0] = 64;
963        config.rows[0] = 16;
964        _tile_loadconfig(config.as_ptr());
965        _tile_zero::<0>();
966        let mat = [1_i8; 1024];
967        _tile_stream_loaddrs::<0>(&mat as *const i8 as *const u8, 64);
968        let mut out = [[0_i8; 64]; 16];
969        _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64);
970        _tile_release();
971        assert_eq!(out, [[1; 64]; 16]);
972    }
973
974    #[simd_test(enable = "amx-avx512,avx10.2")]
975    unsafe fn test_tile_movrow() {
976        _init_amx();
977        let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]);
978
979        let mut config = __tilecfg::default();
980        config.palette = 1;
981        config.colsb[0] = 64;
982        config.rows[0] = 16;
983        _tile_loadconfig(config.as_ptr());
984        _tile_loadd::<0>(array.as_ptr().cast(), 64);
985        for i in 0..16 {
986            let row = _tile_movrow::<0>(i);
987            assert_eq!(*row.as_u8x64().as_array(), [i as _; _]);
988        }
989    }
990
991    #[simd_test(enable = "amx-avx512,avx10.2")]
992    unsafe fn test_tile_cvtrowd2ps() {
993        _init_amx();
994        let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]);
995
996        let mut config = __tilecfg::default();
997        config.palette = 1;
998        config.colsb[0] = 64;
999        config.rows[0] = 16;
1000        _tile_loadconfig(config.as_ptr());
1001        _tile_loadd::<0>(array.as_ptr().cast(), 64);
1002        for i in 0..16 {
1003            let row = _tile_cvtrowd2ps::<0>(i);
1004            assert_eq!(*row.as_f32x16().as_array(), [i as _; _]);
1005        }
1006    }
1007
1008    #[simd_test(enable = "amx-avx512,avx10.2")]
1009    unsafe fn test_tile_cvtrowps2phh() {
1010        _init_amx();
1011        let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);
1012
1013        let mut config = __tilecfg::default();
1014        config.palette = 1;
1015        config.colsb[0] = 64;
1016        config.rows[0] = 16;
1017        _tile_loadconfig(config.as_ptr());
1018        _tile_loadd::<0>(array.as_ptr().cast(), 64);
1019        for i in 0..16 {
1020            let row = _tile_cvtrowps2phh::<0>(i);
1021            assert_eq!(
1022                *row.as_f16x32().as_array(),
1023                array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ })
1024            );
1025        }
1026    }
1027
1028    #[simd_test(enable = "amx-avx512,avx10.2")]
1029    unsafe fn test_tile_cvtrowps2phl() {
1030        _init_amx();
1031        let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);
1032
1033        let mut config = __tilecfg::default();
1034        config.palette = 1;
1035        config.colsb[0] = 64;
1036        config.rows[0] = 16;
1037        _tile_loadconfig(config.as_ptr());
1038        _tile_loadd::<0>(array.as_ptr().cast(), 64);
1039        for i in 0..16 {
1040            let row = _tile_cvtrowps2phl::<0>(i);
1041            assert_eq!(
1042                *row.as_f16x32().as_array(),
1043                array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 })
1044            );
1045        }
1046    }
1047
1048    #[simd_test(enable = "amx-tf32")]
1049    unsafe fn test_tile_mmultf32ps() {
1050        _init_amx();
1051        let a: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);
1052        let b: [[f32; 16]; 16] = [array::from_fn(|j| j as _); _];
1053        let mut res = [[0.0; 16]; 16];
1054
1055        let mut config = __tilecfg::default();
1056        config.palette = 1;
1057        (0..=2).for_each(|i| {
1058            config.colsb[i] = 64;
1059            config.rows[i] = 16;
1060        });
1061        _tile_loadconfig(config.as_ptr());
1062        _tile_zero::<0>();
1063        _tile_loadd::<1>(a.as_ptr().cast(), 64);
1064        _tile_loadd::<2>(b.as_ptr().cast(), 64);
1065        _tile_mmultf32ps::<0, 1, 2>();
1066        _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
1067        _tile_release();
1068
1069        let expected = array::from_fn(|i| array::from_fn(|j| 16.0 * i as f32 * j as f32));
1070        assert_eq!(res, expected);
1071    }
1072}
core/stdarch/crates/core_arch/src/x86_64/amx.rs

core/stdarch/crates/core_arch/src/x86_64/
amx.rs
