Sequential Quantum Gate Decomposer  v1.9.6
Powerful decomposition of general unitarias into one- and two-qubit gates gates
apply_large_kernel_to_input_AVX.cpp
Go to the documentation of this file.
1 /*
2 Created on Fri Jun 26 14:13:26 2020
3 Copyright 2020 Peter Rakyta, Ph.D.
4 
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
8 
9  http://www.apache.org/licenses/LICENSE-2.0
10 
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
16 
17 @author: Peter Rakyta, Ph.D.
18 */
24 #include "apply_large_kernel_to_input_AVX.h"
25 #include "apply_kernel_to_state_vector_input_AVX.h"
26 #include <array>
27 #include "tbb/tbb.h"
28 #include "omp.h"
29 
30 namespace {
31 
32 void apply_2qbit_kernel_to_matrix_input_AVX(Matrix& two_qbit_unitary, Matrix& input, const std::vector<int>& involved_qbits, const int& matrix_size);
33 void apply_2qbit_kernel_to_matrix_input_parallel_AVX_TBB(Matrix& two_qbit_unitary, Matrix& input, const std::vector<int>& involved_qbits, const int& matrix_size);
34 
35 }
36 
43 inline __m256d get_AVX_vector(double* element_outer, double* element_inner){
44 
45  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
46  element_outer_vec = _mm256_permute4x64_pd(element_outer_vec,0b11011000);
47  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
48  element_inner_vec = _mm256_permute4x64_pd(element_inner_vec,0b11011000);
49  __m256d outer_inner_vec = _mm256_shuffle_pd(element_outer_vec,element_inner_vec,0b0000);
50  outer_inner_vec = _mm256_permute4x64_pd(outer_inner_vec,0b11011000);
51 
52  return outer_inner_vec;
53 }
54 
62 inline __m256d complex_mult_AVX(__m256d input_vec, __m256d unitary_row_vec, __m256d neg){
63 
64  __m256d vec3 = _mm256_mul_pd(input_vec, unitary_row_vec);
65  __m256d unitary_row_switched = _mm256_permute_pd(unitary_row_vec, 0x5);
66  unitary_row_switched = _mm256_mul_pd(unitary_row_switched, neg);
67  __m256d vec4 = _mm256_mul_pd(input_vec, unitary_row_switched);
68  __m256d result_vec = _mm256_hsub_pd(vec3, vec4);
69  result_vec = _mm256_permute4x64_pd(result_vec,0b11011000);
70 
71  return result_vec;
72 }
73 
74 template<int n>
75 void apply_fixed_qbit_unitary_AVX(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size);
76 
77 template<int n>
78 void apply_fixed_qbit_unitary_AVX_OpenMP(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size);
79 
80 template<int n>
81 void apply_fixed_qbit_unitary_AVX_TBB(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size);
82 
90 void apply_large_kernel_to_input_AVX(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
91  if (input.cols==1){
92  switch(involved_qbits.size()){
93  case 2:{
94  apply_2qbit_kernel_to_state_vector_input_AVX(unitary, input, involved_qbits, matrix_size);
95  break;
96  }
97  case 3:{
98  apply_3qbit_kernel_to_state_vector_input_AVX(unitary,input,involved_qbits,matrix_size);
99  break;
100  }
101  case 4:{
102  apply_4qbit_kernel_to_state_vector_input_AVX(unitary,input,involved_qbits,matrix_size);
103  break;
104  }
105  case 5:{
106  apply_5qbit_kernel_to_state_vector_input_AVX(unitary,input,involved_qbits,matrix_size);
107  break;
108  }
109  }
110 }
111  else{
112  switch (involved_qbits.size()) {
113  case 2: {
114  apply_2qbit_kernel_to_matrix_input_AVX(unitary, input, involved_qbits, matrix_size);
115  break;
116  }
117  case 3: {
118  apply_fixed_qbit_unitary_AVX<3>(unitary, input, involved_qbits, matrix_size);
119  break;
120  }
121  case 4: {
122  apply_fixed_qbit_unitary_AVX<4>(unitary, input, involved_qbits, matrix_size);
123  break;
124  }
125  case 5: {
126  apply_fixed_qbit_unitary_AVX<5>(unitary, input, involved_qbits, matrix_size);
127  break;
128  }
129  default: {
130  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
131  break;
132  }
133  }
134  }
135 
136 }
137 
145 void apply_large_kernel_to_input_AVX_OpenMP(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
146  if (input.cols==1){
147  switch(involved_qbits.size()){
148  case 2:{
149  apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP(unitary, input, involved_qbits, matrix_size);
150  break;
151  }
152  case 3:{
153  apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP(unitary,input,involved_qbits,matrix_size);
154  break;
155  }
156  case 4:{
157  apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP(unitary,input,involved_qbits,matrix_size);
158  break;
159  }
160  case 5:{
161  apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP(unitary,input,involved_qbits,matrix_size);
162  break;
163  }
164  }
165 }
166  else{
167  switch (involved_qbits.size()) {
168  case 2: {
169  apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP(unitary, input, involved_qbits, matrix_size);
170  break;
171  }
172  case 3: {
173  apply_fixed_qbit_unitary_AVX_OpenMP<3>(unitary, input, involved_qbits, matrix_size);
174  break;
175  }
176  case 4: {
177  apply_fixed_qbit_unitary_AVX_OpenMP<4>(unitary, input, involved_qbits, matrix_size);
178  break;
179  }
180  case 5: {
181  apply_fixed_qbit_unitary_AVX_OpenMP<5>(unitary, input, involved_qbits, matrix_size);
182  break;
183  }
184  default: {
185  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
186  break;
187  }
188  }
189  }
190 
191 }
192 
193 namespace {
194 
195  void apply_2qbit_kernel_to_matrix_input_AVX(Matrix& two_qbit_unitary, Matrix& input, const std::vector<int>& involved_qbits, const int& matrix_size) {
196  int inner_qbit = involved_qbits[0];
197  int outer_qbit = involved_qbits[1];
198  int index_step_outer = 1 << outer_qbit;
199  int index_step_inner = 1 << inner_qbit;
200  int current_idx = 0;
201  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
202 
203  for (int current_idx_pair_outer = current_idx + index_step_outer; current_idx_pair_outer < input.rows; current_idx_pair_outer += (index_step_outer << 1)) {
204  for (int current_idx_inner = 0; current_idx_inner < index_step_outer; current_idx_inner += (index_step_inner << 1)) {
205  for (int idx = 0; idx < index_step_inner; idx++) {
206  int current_idx_outer_loc = current_idx + current_idx_inner + idx;
207  int current_idx_inner_loc = current_idx + current_idx_inner + idx + index_step_inner;
208  int current_idx_outer_pair_loc = current_idx_pair_outer + idx + current_idx_inner;
209  int current_idx_inner_pair_loc = current_idx_pair_outer + idx + current_idx_inner + index_step_inner;
210 
211  int row_offset_outer = current_idx_outer_loc * input.cols;
212  int row_offset_inner = current_idx_inner_loc * input.cols;
213  int row_offset_outer_pair = current_idx_outer_pair_loc * input.cols;
214  int row_offset_inner_pair = current_idx_inner_pair_loc * input.cols;
215 
216  for (int col_idx = 0; col_idx < input.cols; col_idx++) {
217  int current_idx_outer = row_offset_outer + col_idx;
218  int current_idx_inner = row_offset_inner + col_idx;
219  int current_idx_outer_pair = row_offset_outer_pair + col_idx;
220  int current_idx_inner_pair = row_offset_inner_pair + col_idx;
221 
222  double results[8] = {0., 0., 0., 0., 0., 0., 0., 0.};
223 
224  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer;
225  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner;
226  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair;
227  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair;
228 
229  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
230  element_outer_vec = _mm256_permute4x64_pd(element_outer_vec, 0b11011000);
231  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
232  element_inner_vec = _mm256_permute4x64_pd(element_inner_vec, 0b11011000);
233  __m256d outer_inner_vec = _mm256_shuffle_pd(element_outer_vec, element_inner_vec, 0b0000);
234  outer_inner_vec = _mm256_permute4x64_pd(outer_inner_vec, 0b11011000);
235 
236  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
237  element_outer_pair_vec = _mm256_permute4x64_pd(element_outer_pair_vec, 0b11011000);
238  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
239  element_inner_pair_vec = _mm256_permute4x64_pd(element_inner_pair_vec, 0b11011000);
240  __m256d outer_inner_pair_vec = _mm256_shuffle_pd(element_outer_pair_vec, element_inner_pair_vec, 0b0000);
241  outer_inner_pair_vec = _mm256_permute4x64_pd(outer_inner_pair_vec, 0b11011000);
242 
243  for (int mult_idx = 0; mult_idx < 4; mult_idx++) {
244  double* unitary_row_01 = (double*)two_qbit_unitary.get_data() + 8 * mult_idx;
245  double* unitary_row_23 = (double*)two_qbit_unitary.get_data() + 8 * mult_idx + 4;
246 
247  __m256d unitary_row_01_vec = _mm256_loadu_pd(unitary_row_01);
248  __m256d unitary_row_23_vec = _mm256_loadu_pd(unitary_row_23);
249 
250  __m256d result_upper_vec = complex_mult_AVX(outer_inner_vec, unitary_row_01_vec, neg);
251  __m256d result_lower_vec = complex_mult_AVX(outer_inner_pair_vec, unitary_row_23_vec, neg);
252 
253  __m256d result_vec = _mm256_hadd_pd(result_upper_vec, result_lower_vec);
254  result_vec = _mm256_hadd_pd(result_vec, result_vec);
255  double* result = (double*)&result_vec;
256  results[mult_idx * 2] = result[0];
257  results[mult_idx * 2 + 1] = result[2];
258  }
259 
260  input[current_idx_outer].real = results[0];
261  input[current_idx_outer].imag = results[1];
262  input[current_idx_inner].real = results[2];
263  input[current_idx_inner].imag = results[3];
264  input[current_idx_outer_pair].real = results[4];
265  input[current_idx_outer_pair].imag = results[5];
266  input[current_idx_inner_pair].real = results[6];
267  input[current_idx_inner_pair].imag = results[7];
268  }
269  }
270  }
271 
272  current_idx = current_idx + (index_step_outer << 1);
273  }
274 
275  (void)matrix_size;
276  }
277 
278  void apply_2qbit_kernel_to_matrix_input_parallel_AVX_TBB(Matrix& two_qbit_unitary, Matrix& input, const std::vector<int>& involved_qbits, const int& matrix_size) {
279  int inner_qbit = involved_qbits[0];
280  int outer_qbit = involved_qbits[1];
281  int index_step_outer = 1 << outer_qbit;
282  int index_step_inner = 1 << inner_qbit;
283  int current_idx = 0;
284  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
285 
286  for (int current_idx_pair_outer = current_idx + index_step_outer; current_idx_pair_outer < input.rows; current_idx_pair_outer += (index_step_outer << 1)) {
287  for (int current_idx_inner = 0; current_idx_inner < index_step_outer; current_idx_inner += (index_step_inner << 1)) {
288  for (int idx = 0; idx < index_step_inner; idx++) {
289  int current_idx_outer_loc = current_idx + current_idx_inner + idx;
290  int current_idx_inner_loc = current_idx + current_idx_inner + idx + index_step_inner;
291  int current_idx_outer_pair_loc = current_idx_pair_outer + idx + current_idx_inner;
292  int current_idx_inner_pair_loc = current_idx_pair_outer + idx + current_idx_inner + index_step_inner;
293 
294  int row_offset_outer = current_idx_outer_loc * input.cols;
295  int row_offset_inner = current_idx_inner_loc * input.cols;
296  int row_offset_outer_pair = current_idx_outer_pair_loc * input.cols;
297  int row_offset_inner_pair = current_idx_inner_pair_loc * input.cols;
298 
299  tbb::parallel_for(tbb::blocked_range<int>(0, input.cols, 32), [&](const tbb::blocked_range<int>& range) {
300  for (int col_idx = range.begin(); col_idx < range.end(); ++col_idx) {
301  int current_idx_outer = row_offset_outer + col_idx;
302  int current_idx_inner = row_offset_inner + col_idx;
303  int current_idx_outer_pair = row_offset_outer_pair + col_idx;
304  int current_idx_inner_pair = row_offset_inner_pair + col_idx;
305 
306  double results[8] = {0., 0., 0., 0., 0., 0., 0., 0.};
307 
308  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer;
309  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner;
310  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair;
311  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair;
312 
313  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
314  element_outer_vec = _mm256_permute4x64_pd(element_outer_vec, 0b11011000);
315  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
316  element_inner_vec = _mm256_permute4x64_pd(element_inner_vec, 0b11011000);
317  __m256d outer_inner_vec = _mm256_shuffle_pd(element_outer_vec, element_inner_vec, 0b0000);
318  outer_inner_vec = _mm256_permute4x64_pd(outer_inner_vec, 0b11011000);
319 
320  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
321  element_outer_pair_vec = _mm256_permute4x64_pd(element_outer_pair_vec, 0b11011000);
322  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
323  element_inner_pair_vec = _mm256_permute4x64_pd(element_inner_pair_vec, 0b11011000);
324  __m256d outer_inner_pair_vec = _mm256_shuffle_pd(element_outer_pair_vec, element_inner_pair_vec, 0b0000);
325  outer_inner_pair_vec = _mm256_permute4x64_pd(outer_inner_pair_vec, 0b11011000);
326 
327  for (int mult_idx = 0; mult_idx < 4; mult_idx++) {
328  double* unitary_row_01 = (double*)two_qbit_unitary.get_data() + 8 * mult_idx;
329  double* unitary_row_23 = (double*)two_qbit_unitary.get_data() + 8 * mult_idx + 4;
330 
331  __m256d unitary_row_01_vec = _mm256_loadu_pd(unitary_row_01);
332  __m256d unitary_row_23_vec = _mm256_loadu_pd(unitary_row_23);
333 
334  __m256d result_upper_vec = complex_mult_AVX(outer_inner_vec, unitary_row_01_vec, neg);
335  __m256d result_lower_vec = complex_mult_AVX(outer_inner_pair_vec, unitary_row_23_vec, neg);
336 
337  __m256d result_vec = _mm256_hadd_pd(result_upper_vec, result_lower_vec);
338  result_vec = _mm256_hadd_pd(result_vec, result_vec);
339  double* result = (double*)&result_vec;
340  results[mult_idx * 2] = result[0];
341  results[mult_idx * 2 + 1] = result[2];
342  }
343 
344  input[current_idx_outer].real = results[0];
345  input[current_idx_outer].imag = results[1];
346  input[current_idx_inner].real = results[2];
347  input[current_idx_inner].imag = results[3];
348  input[current_idx_outer_pair].real = results[4];
349  input[current_idx_outer_pair].imag = results[5];
350  input[current_idx_inner_pair].real = results[6];
351  input[current_idx_inner_pair].imag = results[7];
352  }
353  });
354  }
355  }
356 
357  current_idx = current_idx + (index_step_outer << 1);
358  }
359 
360  (void)matrix_size;
361  }
362 
363 }
364 
373 void apply_large_kernel_to_input_AVX_TBB(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
374  if (input.cols==1){
375  switch(involved_qbits.size()){
376  case 2:{
377  apply_2qbit_kernel_to_state_vector_input_AVX_TBB(unitary, input, involved_qbits, matrix_size);
378  break;
379  }
380  case 3:{
381  apply_3qbit_kernel_to_state_vector_input_AVX_TBB(unitary,input,involved_qbits,matrix_size);
382  break;
383  }
384  case 4:{
385  apply_4qbit_kernel_to_state_vector_input_AVX_TBB(unitary,input,involved_qbits,matrix_size);
386  break;
387  }
388  case 5:{
389  apply_5qbit_kernel_to_state_vector_input_AVX_TBB(unitary,input,involved_qbits,matrix_size);
390  break;
391  }
392  }
393 }
394  else{
395  switch (involved_qbits.size()) {
396  case 2: {
397  apply_2qbit_kernel_to_matrix_input_parallel_AVX_TBB(unitary, input, involved_qbits, matrix_size);
398  break;
399  }
400  case 3: {
401  apply_fixed_qbit_unitary_AVX_TBB<3>(unitary, input, involved_qbits, matrix_size);
402  break;
403  }
404  case 4: {
405  apply_fixed_qbit_unitary_AVX_TBB<4>(unitary, input, involved_qbits, matrix_size);
406  break;
407  }
408  case 5: {
409  apply_fixed_qbit_unitary_AVX_TBB<5>(unitary, input, involved_qbits, matrix_size);
410  break;
411  }
412  default: {
413  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
414  break;
415  }
416  }
417  }
418 
419 }
426 void precompute_index_mapping(const std::vector<int>& target_qubits,
427  const std::vector<int>& non_targets,
428  std::vector<int>& block_pattern) {
429  int block_size = 1 << target_qubits.size();
430 
431  for (int k = 0; k < block_size; ++k) {
432  int idx = 0;
433  for (size_t bit = 0; bit < target_qubits.size(); ++bit) {
434  if (k & (1 << bit)) {
435  idx |= (1 << target_qubits[bit]);
436  }
437  }
438  block_pattern[k] = idx;
439  }
440 }
441 
451 inline void get_block_indices_fast(int iter_idx,
452  const std::vector<int>& target_qubits,
453  const std::vector<int>& non_targets,
454  const std::vector<int>& block_pattern,
455  std::vector<int>& indices) {
456  int base = 0;
457  for (size_t i = 0; i < non_targets.size(); ++i) {
458  if (iter_idx & (1ULL << i)) {
459  base |= (1 << non_targets[i]);
460  }
461  }
462  for (size_t k = 0; k < block_pattern.size(); ++k) {
463  indices[k] = base | block_pattern[k];
464  }
465 }
466 
481 inline __m256d* construct_mv_xy_vectors(const Matrix& gate_kernel_unitary, const int& matrix_size)
482 {
483  // Allocate aligned memory for AVX (32-byte alignment)
484  __m256d* mv_xy = (__m256d*) _mm_malloc(sizeof(__m256d) * matrix_size * matrix_size, 32);
485 
486  for (int rdx = 0; rdx < matrix_size; rdx++) {
487  for (int cdx = 0; cdx < matrix_size; cdx += 2) {
488 
489  // Precompute both vectors in a single loop
490  mv_xy[rdx * matrix_size + cdx] = _mm256_set_pd(
491  -gate_kernel_unitary[matrix_size*rdx+cdx+1].imag,
492  gate_kernel_unitary[matrix_size*rdx+cdx+1].real,
493  -gate_kernel_unitary[matrix_size*rdx+cdx].imag,
494  gate_kernel_unitary[matrix_size*rdx+cdx].real
495  );
496 
497  mv_xy[rdx * matrix_size + cdx + 1] = _mm256_set_pd(
498  gate_kernel_unitary[matrix_size*rdx+cdx+1].real,
499  gate_kernel_unitary[matrix_size*rdx+cdx+1].imag,
500  gate_kernel_unitary[matrix_size*rdx+cdx].real,
501  gate_kernel_unitary[matrix_size*rdx+cdx].imag
502  );
503  }
504  }
505 
506  return mv_xy;
507 }
508 
516 void apply_2qbit_kernel_to_state_vector_input_AVX_impl(Matrix& two_qbit_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
517  int inner_qbit = involved_qbits[0];
518  int outer_qbit = involved_qbits[1];
519  int index_step_outer = 1 << outer_qbit;
520  int index_step_inner = 1 << inner_qbit;
521  int current_idx = 0;
522 
523 /*
524 AVX kernel developed according to https://github.com/qulacs/qulacs/blob/main/src/csim/update_ops_matrix_dense_single.cpp
525 
526 under MIT License
527 
528 Copyright (c) 2018 Qulacs Authors
529 
530 Permission is hereby granted, free of charge, to any person obtaining a copy
531 of this software and associated documentation files (the "Software"), to deal
532 in the Software without restriction, including without limitation the rights
533 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
534 copies of the Software, and to permit persons to whom the Software is
535 furnished to do so, subject to the following conditions:
536 
537 The above copyright notice and this permission notice shall be included in all
538 copies or substantial portions of the Software.
539 */
540 if (inner_qbit==0){
541  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
542  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
543  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
544  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
545  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
546  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
547  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
548  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
549  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
550  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
551  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
552  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
553  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
554  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
555  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
556  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
557 
558  for (int current_idx = 0; current_idx < input.rows; current_idx += (index_step_outer << 1)) {
559  int current_idx_pair_outer = current_idx + index_step_outer;
560 
561  for (int current_idx_inner = 0; current_idx_inner < index_step_outer; current_idx_inner += (index_step_inner << 1)) {
562 
563  int current_idx_outer_loc = current_idx + current_idx_inner;
564  int current_idx_outer_pair_loc = current_idx_pair_outer + current_idx_inner;
565 
566  // Load two consecutive complex numbers at each base location
567  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
568  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
569 
570  // Load 4 doubles = 2 complex numbers at each location
571  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
572  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
573 
574  // Compute the four matrix-vector products (one for each output element)
575 
576  // Result for current_idx_outer_loc (row 0 of matrix)
577  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
578  __m256d data_u1 = _mm256_mul_pd(element_outer_vec, mv01);
579  __m256d data_u3 = _mm256_mul_pd(element_outer_pair_vec, mv20);
580  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv21);
581  __m256d data_u5 = _mm256_add_pd(data_u3, data_u0);
582  __m256d data_u2 = _mm256_add_pd(data_u1, data_u4);
583  __m256d data_u7 = _mm256_hadd_pd(data_u5, data_u2);
584  __m256d data_u8 = _mm256_permute4x64_pd(data_u7, 0b11011000);
585  __m256d data_u6 = _mm256_hadd_pd(data_u8, data_u8);
586 
587  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
588  __m256d data_d1 = _mm256_mul_pd(element_outer_vec, mv41);
589  __m256d data_d3 = _mm256_mul_pd(element_outer_pair_vec, mv60);
590  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv61);
591  __m256d data_d5 = _mm256_add_pd(data_d3, data_d0);
592  __m256d data_d6 = _mm256_add_pd(data_d1, data_d4);
593  data_d6 = _mm256_hadd_pd(data_d5, data_d6);
594  data_d6 = _mm256_permute4x64_pd(data_d6, 0b11011000);
595  data_d6 = _mm256_hadd_pd(data_d6, data_d6);
596 
597  // Result for row 2 of matrix
598  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
599  __m256d data_e1 = _mm256_mul_pd(element_outer_vec, mv81);
600  __m256d data_e3 = _mm256_mul_pd(element_outer_pair_vec, mv100);
601  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv101);
602  __m256d data_e5 = _mm256_add_pd(data_e3, data_e0);
603  __m256d data_e6 = _mm256_add_pd(data_e1, data_e4);
604  data_e6 = _mm256_hadd_pd(data_e5, data_e6);
605  data_e6 = _mm256_permute4x64_pd(data_e6, 0b11011000);
606  data_e6 = _mm256_hadd_pd(data_e6, data_e6);
607 
608  // Result for row 3 of matrix
609  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
610  __m256d data_f1 = _mm256_mul_pd(element_outer_vec, mv121);
611  __m256d data_f3 = _mm256_mul_pd(element_outer_pair_vec, mv140);
612  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv141);
613  __m256d data_f5 = _mm256_add_pd(data_f3, data_f0);
614  __m256d data_f6 = _mm256_add_pd(data_f1, data_f4);
615  data_f6 = _mm256_hadd_pd(data_f5, data_f6);
616  data_f6 = _mm256_permute4x64_pd(data_f6, 0b11011000);
617  data_f6 = _mm256_hadd_pd(data_f6, data_f6);
618 
619  // Store results back to the same locations where they were loaded
620  __m128d low128u = _mm256_castpd256_pd128(data_u6);
621  __m128d high128u = _mm256_extractf128_pd(data_u6, 1);
622 
623  input[current_idx_outer_loc].real = _mm_cvtsd_f64(low128u);
624  input[current_idx_outer_loc].imag = _mm_cvtsd_f64(high128u);
625 
626  __m128d low128d = _mm256_castpd256_pd128(data_d6);
627  __m128d high128d = _mm256_extractf128_pd(data_d6, 1);
628  input[current_idx_outer_loc + 1].real = _mm_cvtsd_f64(low128d);
629  input[current_idx_outer_loc + 1].imag = _mm_cvtsd_f64(high128d);
630 
631  __m128d low128e = _mm256_castpd256_pd128(data_e6);
632  __m128d high128e = _mm256_extractf128_pd(data_e6, 1);
633  input[current_idx_outer_pair_loc].real = _mm_cvtsd_f64(low128e);
634  input[current_idx_outer_pair_loc].imag = _mm_cvtsd_f64(high128e);
635 
636  __m128d low128f = _mm256_castpd256_pd128(data_f6);
637  __m128d high128f = _mm256_extractf128_pd(data_f6, 1);
638  input[current_idx_outer_pair_loc + 1].real = _mm_cvtsd_f64(low128f);
639  input[current_idx_outer_pair_loc + 1].imag = _mm_cvtsd_f64(high128f);
640  }
641  }
642 }
643  else{
644  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[0].imag, two_qbit_unitary[0].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
645  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[0].real, two_qbit_unitary[0].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
646  __m256d mv10 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[1].imag, two_qbit_unitary[1].real);
647  __m256d mv11 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[1].real, two_qbit_unitary[1].imag);
648  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[2].imag, two_qbit_unitary[2].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
649  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[2].real, two_qbit_unitary[2].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
650  __m256d mv30 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[3].imag, two_qbit_unitary[3].real);
651  __m256d mv31 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[3].real, two_qbit_unitary[3].imag);
652  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[4].imag, two_qbit_unitary[4].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
653  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[4].real, two_qbit_unitary[4].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
654  __m256d mv50 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[5].imag, two_qbit_unitary[5].real);
655  __m256d mv51 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[5].real, two_qbit_unitary[5].imag);
656  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[6].imag, two_qbit_unitary[6].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
657  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[6].real, two_qbit_unitary[6].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
658  __m256d mv70 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[7].imag, two_qbit_unitary[7].real);
659  __m256d mv71 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[7].real, two_qbit_unitary[7].imag);
660  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[8].imag, two_qbit_unitary[8].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
661  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[8].real, two_qbit_unitary[8].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
662  __m256d mv90 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[9].imag, two_qbit_unitary[9].real);
663  __m256d mv91 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[9].real, two_qbit_unitary[9].imag);
664  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[10].imag, two_qbit_unitary[10].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
665  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[10].real, two_qbit_unitary[10].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
666  __m256d mv110 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[11].imag, two_qbit_unitary[11].real);
667  __m256d mv111 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[11].real, two_qbit_unitary[11].imag);
668  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[12].imag, two_qbit_unitary[12].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
669  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[12].real, two_qbit_unitary[12].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
670  __m256d mv130 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[13].imag, two_qbit_unitary[13].real);
671  __m256d mv131 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[13].real, two_qbit_unitary[13].imag);
672  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[14].imag, two_qbit_unitary[14].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
673  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[14].real, two_qbit_unitary[14].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
674  __m256d mv150 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[15].imag, two_qbit_unitary[15].real);
675  __m256d mv151 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[15].real, two_qbit_unitary[15].imag);
676  for (int current_idx_pair_outer=current_idx + index_step_outer; current_idx_pair_outer<input.rows; current_idx_pair_outer=current_idx_pair_outer+(index_step_outer << 1)){
677 
678  for (int current_idx_inner = 0; current_idx_inner < index_step_outer; current_idx_inner=current_idx_inner+(index_step_inner<<1)){
679 
680  for (int idx=0; idx<index_step_inner; idx=idx+2){
681 
682  int current_idx_outer_loc = current_idx + current_idx_inner + idx;
683 
684  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
685  double* element_inner = element_outer + 2 * index_step_inner;
686 
687  double* element_outer_pair = element_outer + 2 * index_step_outer;
688  double* element_inner_pair = element_outer_pair + 2 * index_step_inner;
689 
690 
691  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
692  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
693 
694  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
695  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
696 
697  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
698  __m256d data_u1 = _mm256_mul_pd(element_inner_vec, mv10);
699  __m256d data_u2 = _mm256_mul_pd(element_outer_vec, mv01);
700  __m256d data_u3 = _mm256_mul_pd(element_inner_vec, mv11);
701  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv20);
702  __m256d data_u5 = _mm256_mul_pd(element_inner_pair_vec, mv30);
703  __m256d data_u6 = _mm256_mul_pd(element_outer_pair_vec, mv21);
704  __m256d data_u7 = _mm256_mul_pd(element_inner_pair_vec, mv31);
705  __m256d data_u8 = _mm256_hadd_pd(data_u0, data_u2);
706  __m256d data_u9 = _mm256_hadd_pd(data_u1, data_u3);
707  __m256d data_u10 = _mm256_hadd_pd(data_u4, data_u6);
708  __m256d data_u11 = _mm256_hadd_pd(data_u5, data_u7);
709  __m256d data_u = _mm256_add_pd(data_u8, data_u9);
710  data_u = _mm256_add_pd(data_u, data_u10);
711  data_u = _mm256_add_pd(data_u, data_u11);
712 
713  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
714  __m256d data_d1 = _mm256_mul_pd(element_inner_vec, mv50);
715  __m256d data_d2 = _mm256_mul_pd(element_outer_vec, mv41);
716  __m256d data_d3 = _mm256_mul_pd(element_inner_vec, mv51);
717  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv60);
718  __m256d data_d5 = _mm256_mul_pd(element_inner_pair_vec, mv70);
719  __m256d data_d6 = _mm256_mul_pd(element_outer_pair_vec, mv61);
720  __m256d data_d7 = _mm256_mul_pd(element_inner_pair_vec, mv71);
721  __m256d data_d8 = _mm256_hadd_pd(data_d0, data_d2);
722  __m256d data_d9 = _mm256_hadd_pd(data_d1, data_d3);
723  __m256d data_d10 = _mm256_hadd_pd(data_d4, data_d6);
724  __m256d data_d11 = _mm256_hadd_pd(data_d5, data_d7);
725  __m256d data_d = _mm256_add_pd(data_d8, data_d9);
726  data_d = _mm256_add_pd(data_d, data_d10);
727  data_d = _mm256_add_pd(data_d, data_d11);
728 
729  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
730  __m256d data_e1 = _mm256_mul_pd(element_inner_vec, mv90);
731  __m256d data_e2 = _mm256_mul_pd(element_outer_vec, mv81);
732  __m256d data_e3 = _mm256_mul_pd(element_inner_vec, mv91);
733  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv100);
734  __m256d data_e5 = _mm256_mul_pd(element_inner_pair_vec, mv110);
735  __m256d data_e6 = _mm256_mul_pd(element_outer_pair_vec, mv101);
736  __m256d data_e7 = _mm256_mul_pd(element_inner_pair_vec, mv111);
737  __m256d data_e8 = _mm256_hadd_pd(data_e0, data_e2);
738  __m256d data_e9 = _mm256_hadd_pd(data_e1, data_e3);
739  __m256d data_e10 = _mm256_hadd_pd(data_e4, data_e6);
740  __m256d data_e11 = _mm256_hadd_pd(data_e5, data_e7);
741  __m256d data_e = _mm256_add_pd(data_e8, data_e9);
742  data_e = _mm256_add_pd(data_e, data_e10);
743  data_e = _mm256_add_pd(data_e, data_e11);
744 
745  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
746  __m256d data_f1 = _mm256_mul_pd(element_inner_vec, mv130);
747  __m256d data_f2 = _mm256_mul_pd(element_outer_vec, mv121);
748  __m256d data_f3 = _mm256_mul_pd(element_inner_vec, mv131);
749  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv140);
750  __m256d data_f5 = _mm256_mul_pd(element_inner_pair_vec, mv150);
751  __m256d data_f6 = _mm256_mul_pd(element_outer_pair_vec, mv141);
752  __m256d data_f7 = _mm256_mul_pd(element_inner_pair_vec, mv151);
753  __m256d data_f8 = _mm256_hadd_pd(data_f0, data_f2);
754  __m256d data_f9 = _mm256_hadd_pd(data_f1, data_f3);
755  __m256d data_f10 = _mm256_hadd_pd(data_f4, data_f6);
756  __m256d data_f11 = _mm256_hadd_pd(data_f5, data_f7);
757  __m256d data_f = _mm256_add_pd(data_f8, data_f9);
758  data_f = _mm256_add_pd(data_f, data_f10);
759  data_f = _mm256_add_pd(data_f, data_f11);
760 
761  _mm256_storeu_pd(element_outer, data_u);
762  _mm256_storeu_pd(element_inner, data_d);
763  _mm256_storeu_pd(element_outer_pair, data_e);
764  _mm256_storeu_pd(element_inner_pair, data_f);
765 
766 
767 
768  }
769  }
770  current_idx = current_idx + (index_step_outer << 1);
771  }
772 
773  }
774 
775 }
776 
777 
778 // Macros for 3-qubit row computation when inner_qbit == 0
779 #define CREATE_MATRIX_VECTOR_CONSECUTIVE(base_idx) \
780  __m256d mv##base_idx##0 = _mm256_set_pd(-unitary[base_idx+1].imag, unitary[base_idx+1].real, -unitary[base_idx].imag, unitary[base_idx].real); \
781  __m256d mv##base_idx##1 = _mm256_set_pd( unitary[base_idx+1].real, unitary[base_idx+1].imag, unitary[base_idx].real, unitary[base_idx].imag);
782 
783 #define COMPUTE_3QBIT_ROW_CONSECUTIVE(row_letter, mv00, mv20, mv40, mv60) \
784  __m256d data_real_##row_letter = _mm256_setzero_pd(); \
785  __m256d data_imag_##row_letter = _mm256_setzero_pd(); \
786  data_real_##row_letter = _mm256_fmadd_pd(element_000_vec, mv##mv00##0, data_real_##row_letter); \
787  data_real_##row_letter = _mm256_fmadd_pd(element_010_vec, mv##mv20##0, data_real_##row_letter); \
788  data_real_##row_letter = _mm256_fmadd_pd(element_100_vec, mv##mv40##0, data_real_##row_letter); \
789  data_real_##row_letter = _mm256_fmadd_pd(element_110_vec, mv##mv60##0, data_real_##row_letter); \
790  data_imag_##row_letter = _mm256_fmadd_pd(element_000_vec, mv##mv00##1, data_imag_##row_letter); \
791  data_imag_##row_letter = _mm256_fmadd_pd(element_010_vec, mv##mv20##1, data_imag_##row_letter); \
792  data_imag_##row_letter = _mm256_fmadd_pd(element_100_vec, mv##mv40##1, data_imag_##row_letter); \
793  data_imag_##row_letter = _mm256_fmadd_pd(element_110_vec, mv##mv60##1, data_imag_##row_letter); \
794  __m256d data_##row_letter = _mm256_hadd_pd(data_real_##row_letter, data_imag_##row_letter ); \
795  data_##row_letter = _mm256_permute4x64_pd(data_##row_letter, 0b11011000); \
796  data_##row_letter = _mm256_hadd_pd(data_##row_letter, data_##row_letter); \
797  __m128d low128##row_letter = _mm256_castpd256_pd128(data_##row_letter); \
798  __m128d high128##row_letter = _mm256_extractf128_pd(data_##row_letter, 1); \
799  results[row_idx].real = _mm_cvtsd_f64(low128##row_letter); \
800  results[row_idx].imag = _mm_cvtsd_f64(high128##row_letter);
801 
802 // Macros for 3-qubit row computation when inner_qbit != 0
803 
804 #define CREATE_MATRIX_VECTOR(base_idx) \
805  __m256d mv##base_idx##0 = _mm256_set_pd(-unitary[base_idx].imag, unitary[base_idx].real, -unitary[base_idx].imag, unitary[base_idx].real); \
806  __m256d mv##base_idx##1 = _mm256_set_pd( unitary[base_idx].real, unitary[base_idx].imag, unitary[base_idx].real, unitary[base_idx].imag);
807 
808 #define COMPUTE_3QBIT_ROW(row_letter, base0, base1, base2, base3, base4, base5, base6, base7) \
809  __m256d data_##row_letter##0 = _mm256_mul_pd(element_outer_vec, mv##base0##0); \
810  __m256d data_##row_letter##1 = _mm256_mul_pd(element_inner_vec, mv##base1##0); \
811  __m256d data_##row_letter##2 = _mm256_mul_pd(element_outer_vec, mv##base0##1); \
812  __m256d data_##row_letter##3 = _mm256_mul_pd(element_inner_vec, mv##base1##1); \
813  __m256d data_##row_letter##4 = _mm256_mul_pd(element_middle_vec, mv##base2##0); \
814  __m256d data_##row_letter##5 = _mm256_mul_pd(element_middle_inner_vec, mv##base3##0); \
815  __m256d data_##row_letter##6 = _mm256_mul_pd(element_middle_vec, mv##base2##1); \
816  __m256d data_##row_letter##7 = _mm256_mul_pd(element_middle_inner_vec, mv##base3##1); \
817  __m256d data_##row_letter##8 = _mm256_mul_pd(element_outer_pair_vec, mv##base4##0); \
818  __m256d data_##row_letter##9 = _mm256_mul_pd(element_inner_pair_vec, mv##base5##0); \
819  __m256d data_##row_letter##10 = _mm256_mul_pd(element_outer_pair_vec, mv##base4##1); \
820  __m256d data_##row_letter##11 = _mm256_mul_pd(element_inner_pair_vec, mv##base5##1); \
821  __m256d data_##row_letter##12 = _mm256_mul_pd(element_middle_pair_vec, mv##base6##0); \
822  __m256d data_##row_letter##13 = _mm256_mul_pd(element_middle_inner_pair_vec, mv##base7##0); \
823  __m256d data_##row_letter##14 = _mm256_mul_pd(element_middle_pair_vec, mv##base6##1); \
824  __m256d data_##row_letter##15 = _mm256_mul_pd(element_middle_inner_pair_vec, mv##base7##1); \
825  __m256d data_##row_letter##16 = _mm256_hadd_pd(data_##row_letter##0, data_##row_letter##2); \
826  __m256d data_##row_letter##17 = _mm256_hadd_pd(data_##row_letter##1, data_##row_letter##3); \
827  __m256d data_##row_letter##18 = _mm256_hadd_pd(data_##row_letter##4, data_##row_letter##6); \
828  __m256d data_##row_letter##19 = _mm256_hadd_pd(data_##row_letter##5, data_##row_letter##7); \
829  __m256d data_##row_letter##20 = _mm256_hadd_pd(data_##row_letter##8, data_##row_letter##10); \
830  __m256d data_##row_letter##21 = _mm256_hadd_pd(data_##row_letter##9, data_##row_letter##11); \
831  __m256d data_##row_letter##22 = _mm256_hadd_pd(data_##row_letter##12, data_##row_letter##14); \
832  __m256d data_##row_letter##23 = _mm256_hadd_pd(data_##row_letter##13, data_##row_letter##15); \
833  __m256d data_##row_letter = _mm256_add_pd(data_##row_letter##16, data_##row_letter##17); \
834  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##18); \
835  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##19); \
836  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##20); \
837  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##21); \
838  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##22); \
839  data_##row_letter = _mm256_add_pd(data_##row_letter, data_##row_letter##23);
840 
849 void apply_3qbit_kernel_to_state_vector_input_AVX_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
850  int inner_qbit = involved_qbits[0];
851  int middle_qbit = involved_qbits[1];
852  int outer_qbit = involved_qbits[2];
853 
854  int index_step_inner = 1 << inner_qbit;
855  int index_step_middle = 1 << middle_qbit;
856  int index_step_outer = 1 << outer_qbit;
857 
858  int qubit_num = (int) std::log2(input.rows);
859 
860  std::vector<int> is_target(qubit_num, 0);
861  for (int q : involved_qbits) is_target[q] = 1;
862 
863  std::vector<int> non_targets;
864  non_targets.reserve(qubit_num - 3);
865  for (int q = 0; q < qubit_num; ++q) {
866  if (!is_target[q]) non_targets.push_back(q);
867  }
868 
869  if (inner_qbit == 0) {
870 
871  CREATE_MATRIX_VECTOR_CONSECUTIVE(0) CREATE_MATRIX_VECTOR_CONSECUTIVE(2)
872  CREATE_MATRIX_VECTOR_CONSECUTIVE(4) CREATE_MATRIX_VECTOR_CONSECUTIVE(6)
873  CREATE_MATRIX_VECTOR_CONSECUTIVE(8) CREATE_MATRIX_VECTOR_CONSECUTIVE(10)
874  CREATE_MATRIX_VECTOR_CONSECUTIVE(12) CREATE_MATRIX_VECTOR_CONSECUTIVE(14)
875  CREATE_MATRIX_VECTOR_CONSECUTIVE(16) CREATE_MATRIX_VECTOR_CONSECUTIVE(18)
876  CREATE_MATRIX_VECTOR_CONSECUTIVE(20) CREATE_MATRIX_VECTOR_CONSECUTIVE(22)
877  CREATE_MATRIX_VECTOR_CONSECUTIVE(24) CREATE_MATRIX_VECTOR_CONSECUTIVE(26)
878  CREATE_MATRIX_VECTOR_CONSECUTIVE(28) CREATE_MATRIX_VECTOR_CONSECUTIVE(30)
879  CREATE_MATRIX_VECTOR_CONSECUTIVE(32) CREATE_MATRIX_VECTOR_CONSECUTIVE(34)
880  CREATE_MATRIX_VECTOR_CONSECUTIVE(36) CREATE_MATRIX_VECTOR_CONSECUTIVE(38)
881  CREATE_MATRIX_VECTOR_CONSECUTIVE(40) CREATE_MATRIX_VECTOR_CONSECUTIVE(42)
882  CREATE_MATRIX_VECTOR_CONSECUTIVE(44) CREATE_MATRIX_VECTOR_CONSECUTIVE(46)
883  CREATE_MATRIX_VECTOR_CONSECUTIVE(48) CREATE_MATRIX_VECTOR_CONSECUTIVE(50)
884  CREATE_MATRIX_VECTOR_CONSECUTIVE(52) CREATE_MATRIX_VECTOR_CONSECUTIVE(54)
885  CREATE_MATRIX_VECTOR_CONSECUTIVE(56) CREATE_MATRIX_VECTOR_CONSECUTIVE(58)
886  CREATE_MATRIX_VECTOR_CONSECUTIVE(60) CREATE_MATRIX_VECTOR_CONSECUTIVE(62)
887 
888 
889  for (int iter_idx = 0; iter_idx < matrix_size>>3; iter_idx++) {
890  int base = 0;
891  for (size_t i = 0; i < non_targets.size(); ++i) {
892  if (iter_idx & (1ULL << i)) {
893  base |= (1 << non_targets[i]);
894  }
895  }
896 
897  double* ptr_000 = (double*)input.get_data() + 2 * base;
898  double* ptr_010 = (double*)input.get_data() + 2 * (base | index_step_middle);
899  double* ptr_100 = (double*)input.get_data() + 2 * (base | index_step_outer);
900  double* ptr_110 = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
901 
902  __m256d element_000_vec = _mm256_loadu_pd(ptr_000); // Loads indices[0] and [1]
903  __m256d element_010_vec = _mm256_loadu_pd(ptr_010); // Loads indices[2] and [3]
904  __m256d element_100_vec = _mm256_loadu_pd(ptr_100); // Loads indices[4] and [5]
905  __m256d element_110_vec = _mm256_loadu_pd(ptr_110); // Loads indices[6] and [7]
906 
907  QGD_Complex16 results[8];
908 
909  int row_idx = 0;
910  COMPUTE_3QBIT_ROW_CONSECUTIVE(u, 0, 2, 4, 6)
911  row_idx = 1;
912  COMPUTE_3QBIT_ROW_CONSECUTIVE(d, 8, 10, 12, 14)
913  row_idx = 2;
914  COMPUTE_3QBIT_ROW_CONSECUTIVE(e, 16, 18, 20, 22)
915  row_idx = 3;
916  COMPUTE_3QBIT_ROW_CONSECUTIVE(f, 24, 26, 28, 30)
917  row_idx = 4;
918  COMPUTE_3QBIT_ROW_CONSECUTIVE(g, 32, 34, 36, 38)
919  row_idx = 5;
920  COMPUTE_3QBIT_ROW_CONSECUTIVE(h, 40, 42, 44, 46)
921  row_idx = 6;
922  COMPUTE_3QBIT_ROW_CONSECUTIVE(i, 48, 50, 52, 54)
923  row_idx = 7;
924  COMPUTE_3QBIT_ROW_CONSECUTIVE(j, 56, 58, 60, 62)
925 
926  // Store results at the correct indices
927  input[base] = results[0];
928  input[base+1] = results[1];
929  input[(base | index_step_middle)] = results[2];
930  input[(base | index_step_middle)+1] = results[3];
931  input[(base | index_step_outer)] = results[4];
932  input[(base | index_step_outer)+1] = results[5];
933  input[(base | index_step_middle | index_step_outer)] = results[6];
934  input[(base | index_step_middle | index_step_outer)+1] = results[7];
935  }
936  }
937  else {
938  // Non-consecutive case
939  CREATE_MATRIX_VECTOR(0) CREATE_MATRIX_VECTOR(1) CREATE_MATRIX_VECTOR(2) CREATE_MATRIX_VECTOR(3)
940  CREATE_MATRIX_VECTOR(4) CREATE_MATRIX_VECTOR(5) CREATE_MATRIX_VECTOR(6) CREATE_MATRIX_VECTOR(7)
941  CREATE_MATRIX_VECTOR(8) CREATE_MATRIX_VECTOR(9) CREATE_MATRIX_VECTOR(10) CREATE_MATRIX_VECTOR(11)
942  CREATE_MATRIX_VECTOR(12) CREATE_MATRIX_VECTOR(13) CREATE_MATRIX_VECTOR(14) CREATE_MATRIX_VECTOR(15)
943  CREATE_MATRIX_VECTOR(16) CREATE_MATRIX_VECTOR(17) CREATE_MATRIX_VECTOR(18) CREATE_MATRIX_VECTOR(19)
944  CREATE_MATRIX_VECTOR(20) CREATE_MATRIX_VECTOR(21) CREATE_MATRIX_VECTOR(22) CREATE_MATRIX_VECTOR(23)
945  CREATE_MATRIX_VECTOR(24) CREATE_MATRIX_VECTOR(25) CREATE_MATRIX_VECTOR(26) CREATE_MATRIX_VECTOR(27)
946  CREATE_MATRIX_VECTOR(28) CREATE_MATRIX_VECTOR(29) CREATE_MATRIX_VECTOR(30) CREATE_MATRIX_VECTOR(31)
947  CREATE_MATRIX_VECTOR(32) CREATE_MATRIX_VECTOR(33) CREATE_MATRIX_VECTOR(34) CREATE_MATRIX_VECTOR(35)
948  CREATE_MATRIX_VECTOR(36) CREATE_MATRIX_VECTOR(37) CREATE_MATRIX_VECTOR(38) CREATE_MATRIX_VECTOR(39)
949  CREATE_MATRIX_VECTOR(40) CREATE_MATRIX_VECTOR(41) CREATE_MATRIX_VECTOR(42) CREATE_MATRIX_VECTOR(43)
950  CREATE_MATRIX_VECTOR(44) CREATE_MATRIX_VECTOR(45) CREATE_MATRIX_VECTOR(46) CREATE_MATRIX_VECTOR(47)
951  CREATE_MATRIX_VECTOR(48) CREATE_MATRIX_VECTOR(49) CREATE_MATRIX_VECTOR(50) CREATE_MATRIX_VECTOR(51)
952  CREATE_MATRIX_VECTOR(52) CREATE_MATRIX_VECTOR(53) CREATE_MATRIX_VECTOR(54) CREATE_MATRIX_VECTOR(55)
953  CREATE_MATRIX_VECTOR(56) CREATE_MATRIX_VECTOR(57) CREATE_MATRIX_VECTOR(58) CREATE_MATRIX_VECTOR(59)
954  CREATE_MATRIX_VECTOR(60) CREATE_MATRIX_VECTOR(61) CREATE_MATRIX_VECTOR(62) CREATE_MATRIX_VECTOR(63)
955 
956  for (int iter_idx = 0; iter_idx < matrix_size>>3; iter_idx+=2) {
957 
958  int base = 0;
959  for (size_t i = 0; i < non_targets.size(); ++i) {
960  if (iter_idx & (1ULL << i)) {
961  base |= (1 << non_targets[i]);
962  }
963  }
964 
965  double* element_outer = (double*)input.get_data() + 2 * base;
966  double* element_inner = (double*)input.get_data() + 2 * (base | index_step_inner);
967  double* element_middle = (double*)input.get_data() + 2 * (base | index_step_middle);
968  double* element_middle_inner = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle);
969  double* element_outer_pair = (double*)input.get_data() + 2 * (base | index_step_outer);
970  double* element_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_outer);
971  double* element_middle_pair = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
972  double* element_middle_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle | index_step_outer);
973 
974  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
975  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
976  __m256d element_middle_vec = _mm256_loadu_pd(element_middle);
977  __m256d element_middle_inner_vec = _mm256_loadu_pd(element_middle_inner);
978  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
979  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
980  __m256d element_middle_pair_vec = _mm256_loadu_pd(element_middle_pair);
981  __m256d element_middle_inner_pair_vec = _mm256_loadu_pd(element_middle_inner_pair);
982 
983  // Compute all 8 rows using macros
984  COMPUTE_3QBIT_ROW(u, 0, 1, 2, 3, 4, 5, 6, 7)
985  COMPUTE_3QBIT_ROW(d, 8, 9, 10, 11, 12, 13, 14, 15)
986  COMPUTE_3QBIT_ROW(e, 16, 17, 18, 19, 20, 21, 22, 23)
987  COMPUTE_3QBIT_ROW(f, 24, 25, 26, 27, 28, 29, 30, 31)
988  COMPUTE_3QBIT_ROW(g, 32, 33, 34, 35, 36, 37, 38, 39)
989  COMPUTE_3QBIT_ROW(h, 40, 41, 42, 43, 44, 45, 46, 47)
990  COMPUTE_3QBIT_ROW(i, 48, 49, 50, 51, 52, 53, 54, 55)
991  COMPUTE_3QBIT_ROW(j, 56, 57, 58, 59, 60, 61, 62, 63)
992 
993  // Store results
994  _mm256_storeu_pd(element_outer, data_u);
995  _mm256_storeu_pd(element_inner, data_d);
996  _mm256_storeu_pd(element_middle, data_e);
997  _mm256_storeu_pd(element_middle_inner, data_f);
998  _mm256_storeu_pd(element_outer_pair, data_g);
999  _mm256_storeu_pd(element_inner_pair, data_h);
1000  _mm256_storeu_pd(element_middle_pair, data_i);
1001  _mm256_storeu_pd(element_middle_inner_pair, data_j);
1002  }
1003  }
1004 }
1005 
1013 void apply_4qbit_kernel_to_state_vector_input_AVX_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
1014 
1015  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
1016 
1017  int index_step_inner = 1 << involved_qbits[0];
1018  int index_step_middle1 = 1 << involved_qbits[1];
1019  int index_step_middle2 = 1 << involved_qbits[2];
1020  int index_step_outer = 1 << involved_qbits[3];
1021 
1022  // Properly iterate through all blocks
1023  int num_qubits = (int)std::log2(matrix_size);
1024  int num_blocks = matrix_size >> 4; // 2^4 = 16 elements per block
1025 
1026 
1027  // Identify non-involved qubits
1028  std::vector<int> is_target(num_qubits, 0);
1029  for (int q : involved_qbits) is_target[q] = 1;
1030 
1031  std::vector<int> non_targets;
1032  non_targets.reserve(num_qubits - 4);
1033  for (int q = 0; q < num_qubits; ++q) {
1034  if (!is_target[q]) non_targets.push_back(q);
1035  }
1036 
1037  for (int block_idx = 0; block_idx < num_blocks; block_idx++) {
1038  __m256d element_0000_vec_real = _mm256_setzero_pd();
1039  __m256d element_0010_vec_real = _mm256_setzero_pd();
1040  __m256d element_0100_vec_real = _mm256_setzero_pd();
1041  __m256d element_0110_vec_real = _mm256_setzero_pd();
1042  __m256d element_1000_vec_real = _mm256_setzero_pd();
1043  __m256d element_1010_vec_real = _mm256_setzero_pd();
1044  __m256d element_1100_vec_real = _mm256_setzero_pd();
1045  __m256d element_1110_vec_real = _mm256_setzero_pd();
1046 
1047  int base = 0;
1048  for (size_t i = 0; i < non_targets.size(); ++i) {
1049  if (block_idx & (1ULL << i)) {
1050  base |= (1 << non_targets[i]);
1051  }
1052  }
1053 
1054  int current_idx_outer_loc = base;
1055  int current_idx_inner_loc = base | index_step_inner;
1056  int current_idx_middle1_loc = base | index_step_middle1;
1057  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
1058  int current_idx_middle2_loc = base | index_step_middle2;
1059  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
1060  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
1061  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
1062 
1063  int current_idx_outer_pair_loc = base | index_step_outer;
1064  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
1065  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
1066  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
1067  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
1068  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
1069  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
1070  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
1071  if(involved_qbits[0] == 0){
1072  //preload all 16 elements instead of the unitary kernel matrix
1073  element_0000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
1074  element_0010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
1075  element_0100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
1076  element_0110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
1077  element_1000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
1078  element_1010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
1079  element_1100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
1080  element_1110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
1081  }
1082  else{
1083 
1084  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
1085  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
1086 
1087  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
1088  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
1089 
1090  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
1091  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
1092 
1093  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
1094  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
1095 
1096  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
1097  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
1098 
1099  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
1100  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
1101 
1102  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
1103  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
1104 
1105  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
1106  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
1107 
1108  element_0000_vec_real = get_AVX_vector(element_outer, element_inner);
1109  element_0010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
1110  element_0100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
1111  element_0110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
1112  element_1000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
1113  element_1010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
1114  element_1100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
1115  element_1110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
1116  }
1117 
1118  __m256d element_0000_vec_imag = _mm256_permute4x64_pd(element_0000_vec_real, 0b10110001);
1119  __m256d element_0010_vec_imag = _mm256_permute4x64_pd(element_0010_vec_real, 0b10110001);
1120  __m256d element_0100_vec_imag = _mm256_permute4x64_pd(element_0100_vec_real, 0b10110001);
1121  __m256d element_0110_vec_imag = _mm256_permute4x64_pd(element_0110_vec_real, 0b10110001);
1122  __m256d element_1000_vec_imag = _mm256_permute4x64_pd(element_1000_vec_real, 0b10110001);
1123  __m256d element_1010_vec_imag = _mm256_permute4x64_pd(element_1010_vec_real, 0b10110001);
1124  __m256d element_1100_vec_imag = _mm256_permute4x64_pd(element_1100_vec_real, 0b10110001);
1125  __m256d element_1110_vec_imag = _mm256_permute4x64_pd(element_1110_vec_real, 0b10110001);
1126 
1127  element_0000_vec_real = _mm256_mul_pd(element_0000_vec_real,neg);
1128  element_0010_vec_real = _mm256_mul_pd(element_0010_vec_real,neg);
1129  element_0100_vec_real = _mm256_mul_pd(element_0100_vec_real,neg);
1130  element_0110_vec_real = _mm256_mul_pd(element_0110_vec_real,neg);
1131  element_1000_vec_real = _mm256_mul_pd(element_1000_vec_real,neg);
1132  element_1010_vec_real = _mm256_mul_pd(element_1010_vec_real,neg);
1133  element_1100_vec_real = _mm256_mul_pd(element_1100_vec_real,neg);
1134  element_1110_vec_real = _mm256_mul_pd(element_1110_vec_real,neg);
1135 
1136  QGD_Complex16 results[16];
1137  for (int mult_idx = 0; mult_idx < 16; mult_idx++) {
1138  double* unitary_row_1 = (double*)unitary.get_data() + 32*mult_idx;
1139  double* unitary_row_2 = unitary_row_1 + 4;
1140  double* unitary_row_3 = unitary_row_1 + 8;
1141  double* unitary_row_4 = unitary_row_1 + 12;
1142  double* unitary_row_5 = unitary_row_1 + 16;
1143  double* unitary_row_6 = unitary_row_1 + 20;
1144  double* unitary_row_7 = unitary_row_1 + 24;
1145  double* unitary_row_8 = unitary_row_1 + 28;
1146 
1147  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
1148  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
1149  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
1150  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
1151  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
1152  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
1153  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
1154  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
1155 
1156  __m256d data_real = _mm256_setzero_pd();
1157  __m256d data_imag = _mm256_setzero_pd();
1158 
1159  data_real = _mm256_fmadd_pd(element_0000_vec_real, row1_vec, data_real);
1160  data_imag = _mm256_fmadd_pd(element_0000_vec_imag, row1_vec, data_imag);
1161  data_real = _mm256_fmadd_pd(element_0010_vec_real, row2_vec, data_real);
1162  data_imag = _mm256_fmadd_pd(element_0010_vec_imag, row2_vec, data_imag);
1163  data_real = _mm256_fmadd_pd(element_0100_vec_real, row3_vec, data_real);
1164  data_imag = _mm256_fmadd_pd(element_0100_vec_imag, row3_vec, data_imag);
1165  data_real = _mm256_fmadd_pd(element_0110_vec_real, row4_vec, data_real);
1166  data_imag = _mm256_fmadd_pd(element_0110_vec_imag, row4_vec, data_imag);
1167  data_real = _mm256_fmadd_pd(element_1000_vec_real, row5_vec, data_real);
1168  data_imag = _mm256_fmadd_pd(element_1000_vec_imag, row5_vec, data_imag);
1169  data_real = _mm256_fmadd_pd(element_1010_vec_real, row6_vec, data_real);
1170  data_imag = _mm256_fmadd_pd(element_1010_vec_imag, row6_vec, data_imag);
1171  data_real = _mm256_fmadd_pd(element_1100_vec_real, row7_vec, data_real);
1172  data_imag = _mm256_fmadd_pd(element_1100_vec_imag, row7_vec, data_imag);
1173  data_real = _mm256_fmadd_pd(element_1110_vec_real, row8_vec, data_real);
1174  data_imag = _mm256_fmadd_pd(element_1110_vec_imag, row8_vec, data_imag);
1175 
1176  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
1177  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
1178  final_vec = _mm256_hadd_pd(final_vec, final_vec);
1179  __m128d low128 = _mm256_castpd256_pd128(final_vec);
1180  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
1181  results[mult_idx].real = _mm_cvtsd_f64(low128);
1182  results[mult_idx].imag = _mm_cvtsd_f64(high128);
1183 
1184  }
1185  input[current_idx_outer_loc] = results[0];
1186  input[current_idx_inner_loc] = results[1];
1187  input[current_idx_middle1_loc] = results[2];
1188  input[current_idx_middle1_inner_loc] = results[3];
1189  input[current_idx_middle2_loc] = results[4];
1190  input[current_idx_middle2_inner_loc] = results[5];
1191  input[current_idx_middle12_loc] = results[6];
1192  input[current_idx_middle12_inner_loc] = results[7];
1193  input[current_idx_outer_pair_loc] = results[8];
1194  input[current_idx_inner_pair_loc] = results[9];
1195  input[current_idx_middle1_pair_loc] = results[10];
1196  input[current_idx_middle1_inner_pair_loc] = results[11];
1197  input[current_idx_middle2_pair_loc] = results[12];
1198  input[current_idx_middle2_inner_pair_loc] = results[13];
1199  input[current_idx_middle12_pair_loc] = results[14];
1200  input[current_idx_middle12_inner_pair_loc] = results[15];
1201 
1202 
1203 
1204  }
1205 
1206 }
1207 
1216 void apply_5qbit_kernel_to_state_vector_input_AVX_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
1217  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
1218 
1219  int index_step_inner = 1 << involved_qbits[0];
1220  int index_step_middle1 = 1 << involved_qbits[1];
1221  int index_step_middle2 = 1 << involved_qbits[2];
1222  int index_step_middle3 = 1 << involved_qbits[3];
1223  int index_step_outer = 1 << involved_qbits[4];
1224 
1225  int num_qubits = (int)std::log2(matrix_size);
1226  int num_blocks = matrix_size >> 5;
1227  std::vector<int> is_target(num_qubits, 0);
1228  for (int q : involved_qbits) is_target[q] = 1;
1229 
1230  std::vector<int> non_targets;
1231  non_targets.reserve(num_qubits - 5);
1232  for (int q = 0; q < num_qubits; ++q) {
1233  if (!is_target[q]) non_targets.push_back(q);
1234  }
1235 
1236  for (int block_idx = 0; block_idx < num_blocks; block_idx++) {
1237  // Calculate base index properly
1238  int base = 0;
1239  for (size_t i = 0; i < non_targets.size(); ++i) {
1240  if (block_idx & (1ULL << i)) {
1241  base |= (1 << non_targets[i]);
1242  }
1243  }
1244  // Calculate all 32 indices using OR operations
1245  int current_idx_outer_loc = base;
1246  int current_idx_inner_loc = base | index_step_inner;
1247 
1248  int current_idx_middle1_loc = base | index_step_middle1;
1249  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
1250 
1251  int current_idx_middle2_loc = base | index_step_middle2;
1252  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
1253 
1254  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
1255  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
1256 
1257  int current_idx_middle3_loc = base | index_step_middle3;
1258  int current_idx_middle3_inner_loc = base | index_step_middle3 | index_step_inner;
1259 
1260  int current_idx_middle13_loc = base | index_step_middle1 | index_step_middle3;
1261  int current_idx_middle13_inner_loc = base | index_step_middle1 | index_step_middle3 | index_step_inner;
1262 
1263  int current_idx_middle23_loc = base | index_step_middle2 | index_step_middle3;
1264  int current_idx_middle23_inner_loc = base | index_step_middle2 | index_step_middle3 | index_step_inner;
1265 
1266  int current_idx_middle123_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3;
1267  int current_idx_middle123_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
1268 
1269  int current_idx_outer_pair_loc = base | index_step_outer;
1270  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
1271 
1272  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
1273  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
1274 
1275  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
1276  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
1277 
1278  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
1279  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
1280 
1281  int current_idx_middle3_pair_loc = base | index_step_outer | index_step_middle3;
1282  int current_idx_middle3_inner_pair_loc = base | index_step_outer | index_step_middle3 | index_step_inner;
1283 
1284  int current_idx_middle13_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3;
1285  int current_idx_middle13_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3 | index_step_inner;
1286 
1287  int current_idx_middle23_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3;
1288  int current_idx_middle23_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3 | index_step_inner;
1289 
1290  int current_idx_middle123_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3;
1291  int current_idx_middle123_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
1292 
1293  __m256d element_00000_vec_real = _mm256_setzero_pd();
1294  __m256d element_00010_vec_real = _mm256_setzero_pd();
1295  __m256d element_00100_vec_real = _mm256_setzero_pd();
1296  __m256d element_00110_vec_real = _mm256_setzero_pd();
1297  __m256d element_01000_vec_real = _mm256_setzero_pd();
1298  __m256d element_01010_vec_real = _mm256_setzero_pd();
1299  __m256d element_01100_vec_real = _mm256_setzero_pd();
1300  __m256d element_01110_vec_real = _mm256_setzero_pd();
1301  __m256d element_10000_vec_real = _mm256_setzero_pd();
1302  __m256d element_10010_vec_real = _mm256_setzero_pd();
1303  __m256d element_10100_vec_real = _mm256_setzero_pd();
1304  __m256d element_10110_vec_real = _mm256_setzero_pd();
1305  __m256d element_11000_vec_real = _mm256_setzero_pd();
1306  __m256d element_11010_vec_real = _mm256_setzero_pd();
1307  __m256d element_11100_vec_real = _mm256_setzero_pd();
1308  __m256d element_11110_vec_real = _mm256_setzero_pd();
1309 
1310  if (involved_qbits[0] == 0) {
1311  // Preload all 32 elements instead of the unitary kernel matrix
1312  element_00000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
1313  element_00010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
1314  element_00100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
1315  element_00110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
1316  element_01000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_loc);
1317  element_01010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_loc);
1318  element_01100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_loc);
1319  element_01110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_loc);
1320  element_10000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
1321  element_10010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
1322  element_10100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
1323  element_10110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
1324  element_11000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_pair_loc);
1325  element_11010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_pair_loc);
1326  element_11100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_pair_loc);
1327  element_11110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_pair_loc);
1328 
1329  }
1330  else {
1331  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
1332  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
1333 
1334  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
1335  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
1336 
1337  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
1338  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
1339 
1340  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
1341  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
1342 
1343  double* element_middle3 = (double*)input.get_data() + 2 * current_idx_middle3_loc;
1344  double* element_middle3_inner = (double*)input.get_data() + 2 * current_idx_middle3_inner_loc;
1345 
1346  double* element_middle13 = (double*)input.get_data() + 2 * current_idx_middle13_loc;
1347  double* element_middle13_inner = (double*)input.get_data() + 2 * current_idx_middle13_inner_loc;
1348 
1349  double* element_middle23 = (double*)input.get_data() + 2 * current_idx_middle23_loc;
1350  double* element_middle23_inner = (double*)input.get_data() + 2 * current_idx_middle23_inner_loc;
1351 
1352  double* element_middle123 = (double*)input.get_data() + 2 * current_idx_middle123_loc;
1353  double* element_middle123_inner = (double*)input.get_data() + 2 * current_idx_middle123_inner_loc;
1354 
1355  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
1356  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
1357 
1358  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
1359  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
1360 
1361  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
1362  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
1363 
1364  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
1365  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
1366 
1367  double* element_middle3_pair = (double*)input.get_data() + 2 * current_idx_middle3_pair_loc;
1368  double* element_middle3_inner_pair = (double*)input.get_data() + 2 * current_idx_middle3_inner_pair_loc;
1369 
1370  double* element_middle13_pair = (double*)input.get_data() + 2 * current_idx_middle13_pair_loc;
1371  double* element_middle13_inner_pair = (double*)input.get_data() + 2 * current_idx_middle13_inner_pair_loc;
1372 
1373  double* element_middle23_pair = (double*)input.get_data() + 2 * current_idx_middle23_pair_loc;
1374  double* element_middle23_inner_pair = (double*)input.get_data() + 2 * current_idx_middle23_inner_pair_loc;
1375 
1376  double* element_middle123_pair = (double*)input.get_data() + 2 * current_idx_middle123_pair_loc;
1377  double* element_middle123_inner_pair = (double*)input.get_data() + 2 * current_idx_middle123_inner_pair_loc;
1378 
1379  element_00000_vec_real = get_AVX_vector(element_outer, element_inner);
1380  element_00010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
1381  element_00100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
1382  element_00110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
1383  element_01000_vec_real = get_AVX_vector(element_middle3, element_middle3_inner);
1384  element_01010_vec_real = get_AVX_vector(element_middle13, element_middle13_inner);
1385  element_01100_vec_real = get_AVX_vector(element_middle23, element_middle23_inner);
1386  element_01110_vec_real = get_AVX_vector(element_middle123, element_middle123_inner);
1387  element_10000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
1388  element_10010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
1389  element_10100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
1390  element_10110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
1391  element_11000_vec_real = get_AVX_vector(element_middle3_pair, element_middle3_inner_pair);
1392  element_11010_vec_real = get_AVX_vector(element_middle13_pair, element_middle13_inner_pair);
1393  element_11100_vec_real = get_AVX_vector(element_middle23_pair, element_middle23_inner_pair);
1394  element_11110_vec_real = get_AVX_vector(element_middle123_pair, element_middle123_inner_pair);
1395 
1396  }
1397  __m256d element_00000_vec_imag = _mm256_permute4x64_pd(element_00000_vec_real, 0b10110001);
1398  __m256d element_00010_vec_imag = _mm256_permute4x64_pd(element_00010_vec_real, 0b10110001);
1399  __m256d element_00100_vec_imag = _mm256_permute4x64_pd(element_00100_vec_real, 0b10110001);
1400  __m256d element_00110_vec_imag = _mm256_permute4x64_pd(element_00110_vec_real, 0b10110001);
1401  __m256d element_01000_vec_imag = _mm256_permute4x64_pd(element_01000_vec_real, 0b10110001);
1402  __m256d element_01010_vec_imag = _mm256_permute4x64_pd(element_01010_vec_real, 0b10110001);
1403  __m256d element_01100_vec_imag = _mm256_permute4x64_pd(element_01100_vec_real, 0b10110001);
1404  __m256d element_01110_vec_imag = _mm256_permute4x64_pd(element_01110_vec_real, 0b10110001);
1405  __m256d element_10000_vec_imag = _mm256_permute4x64_pd(element_10000_vec_real, 0b10110001);
1406  __m256d element_10010_vec_imag = _mm256_permute4x64_pd(element_10010_vec_real, 0b10110001);
1407  __m256d element_10100_vec_imag = _mm256_permute4x64_pd(element_10100_vec_real, 0b10110001);
1408  __m256d element_10110_vec_imag = _mm256_permute4x64_pd(element_10110_vec_real, 0b10110001);
1409  __m256d element_11000_vec_imag = _mm256_permute4x64_pd(element_11000_vec_real, 0b10110001);
1410  __m256d element_11010_vec_imag = _mm256_permute4x64_pd(element_11010_vec_real, 0b10110001);
1411  __m256d element_11100_vec_imag = _mm256_permute4x64_pd(element_11100_vec_real, 0b10110001);
1412  __m256d element_11110_vec_imag = _mm256_permute4x64_pd(element_11110_vec_real, 0b10110001);
1413 
1414  element_00000_vec_real = _mm256_mul_pd(element_00000_vec_real,neg);
1415  element_00010_vec_real = _mm256_mul_pd(element_00010_vec_real,neg);
1416  element_00100_vec_real = _mm256_mul_pd(element_00100_vec_real,neg);
1417  element_00110_vec_real = _mm256_mul_pd(element_00110_vec_real,neg);
1418  element_01000_vec_real = _mm256_mul_pd(element_01000_vec_real,neg);
1419  element_01010_vec_real = _mm256_mul_pd(element_01010_vec_real,neg);
1420  element_01100_vec_real = _mm256_mul_pd(element_01100_vec_real,neg);
1421  element_01110_vec_real = _mm256_mul_pd(element_01110_vec_real,neg);
1422  element_10000_vec_real = _mm256_mul_pd(element_10000_vec_real,neg);
1423  element_10010_vec_real = _mm256_mul_pd(element_10010_vec_real,neg);
1424  element_10100_vec_real = _mm256_mul_pd(element_10100_vec_real,neg);
1425  element_10110_vec_real = _mm256_mul_pd(element_10110_vec_real,neg);
1426  element_11000_vec_real = _mm256_mul_pd(element_11000_vec_real,neg);
1427  element_11010_vec_real = _mm256_mul_pd(element_11010_vec_real,neg);
1428  element_11100_vec_real = _mm256_mul_pd(element_11100_vec_real,neg);
1429  element_11110_vec_real = _mm256_mul_pd(element_11110_vec_real,neg);
1430 
1431  QGD_Complex16 results[32];
1432  for (int mult_idx = 0; mult_idx < 32; mult_idx++) {
1433  double* unitary_row_1 = (double*)unitary.get_data() + 64*mult_idx;
1434  double* unitary_row_2 = unitary_row_1 + 4;
1435  double* unitary_row_3 = unitary_row_1 + 8;
1436  double* unitary_row_4 = unitary_row_1 + 12;
1437  double* unitary_row_5 = unitary_row_1 + 16;
1438  double* unitary_row_6 = unitary_row_1 + 20;
1439  double* unitary_row_7 = unitary_row_1 + 24;
1440  double* unitary_row_8 = unitary_row_1 + 28;
1441  double* unitary_row_9 = unitary_row_1 + 32;
1442  double* unitary_row_10 = unitary_row_1 + 36;
1443  double* unitary_row_11 = unitary_row_1 + 40;
1444  double* unitary_row_12 = unitary_row_1 + 44;
1445  double* unitary_row_13 = unitary_row_1 + 48;
1446  double* unitary_row_14 = unitary_row_1 + 52;
1447  double* unitary_row_15 = unitary_row_1 + 56;
1448  double* unitary_row_16 = unitary_row_1 + 60;
1449 
1450  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
1451  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
1452  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
1453  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
1454  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
1455  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
1456  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
1457  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
1458  __m256d row9_vec = _mm256_loadu_pd(unitary_row_9);
1459  __m256d row10_vec = _mm256_loadu_pd(unitary_row_10);
1460  __m256d row11_vec = _mm256_loadu_pd(unitary_row_11);
1461  __m256d row12_vec = _mm256_loadu_pd(unitary_row_12);
1462  __m256d row13_vec = _mm256_loadu_pd(unitary_row_13);
1463  __m256d row14_vec = _mm256_loadu_pd(unitary_row_14);
1464  __m256d row15_vec = _mm256_loadu_pd(unitary_row_15);
1465  __m256d row16_vec = _mm256_loadu_pd(unitary_row_16);
1466 
1467  __m256d data_real = _mm256_setzero_pd();
1468  __m256d data_imag = _mm256_setzero_pd();
1469 
1470  data_real = _mm256_fmadd_pd(element_00000_vec_real, row1_vec, data_real);
1471  data_imag = _mm256_fmadd_pd(element_00000_vec_imag, row1_vec, data_imag);
1472  data_real = _mm256_fmadd_pd(element_00010_vec_real, row2_vec, data_real);
1473  data_imag = _mm256_fmadd_pd(element_00010_vec_imag, row2_vec, data_imag);
1474  data_real = _mm256_fmadd_pd(element_00100_vec_real, row3_vec, data_real);
1475  data_imag = _mm256_fmadd_pd(element_00100_vec_imag, row3_vec, data_imag);
1476  data_real = _mm256_fmadd_pd(element_00110_vec_real, row4_vec, data_real);
1477  data_imag = _mm256_fmadd_pd(element_00110_vec_imag, row4_vec, data_imag);
1478  data_real = _mm256_fmadd_pd(element_01000_vec_real, row5_vec, data_real);
1479  data_imag = _mm256_fmadd_pd(element_01000_vec_imag, row5_vec, data_imag);
1480  data_real = _mm256_fmadd_pd(element_01010_vec_real, row6_vec, data_real);
1481  data_imag = _mm256_fmadd_pd(element_01010_vec_imag, row6_vec, data_imag);
1482  data_real = _mm256_fmadd_pd(element_01100_vec_real, row7_vec, data_real);
1483  data_imag = _mm256_fmadd_pd(element_01100_vec_imag, row7_vec, data_imag);
1484  data_real = _mm256_fmadd_pd(element_01110_vec_real, row8_vec, data_real);
1485  data_imag = _mm256_fmadd_pd(element_01110_vec_imag, row8_vec, data_imag);
1486  data_real = _mm256_fmadd_pd(element_10000_vec_real, row9_vec, data_real);
1487  data_imag = _mm256_fmadd_pd(element_10000_vec_imag, row9_vec, data_imag);
1488  data_real = _mm256_fmadd_pd(element_10010_vec_real, row10_vec, data_real);
1489  data_imag = _mm256_fmadd_pd(element_10010_vec_imag, row10_vec, data_imag);
1490  data_real = _mm256_fmadd_pd(element_10100_vec_real, row11_vec, data_real);
1491  data_imag = _mm256_fmadd_pd(element_10100_vec_imag, row11_vec, data_imag);
1492  data_real = _mm256_fmadd_pd(element_10110_vec_real, row12_vec, data_real);
1493  data_imag = _mm256_fmadd_pd(element_10110_vec_imag, row12_vec, data_imag);
1494  data_real = _mm256_fmadd_pd(element_11000_vec_real, row13_vec, data_real);
1495  data_imag = _mm256_fmadd_pd(element_11000_vec_imag, row13_vec, data_imag);
1496  data_real = _mm256_fmadd_pd(element_11010_vec_real, row14_vec, data_real);
1497  data_imag = _mm256_fmadd_pd(element_11010_vec_imag, row14_vec, data_imag);
1498  data_real = _mm256_fmadd_pd(element_11100_vec_real, row15_vec, data_real);
1499  data_imag = _mm256_fmadd_pd(element_11100_vec_imag, row15_vec, data_imag);
1500  data_real = _mm256_fmadd_pd(element_11110_vec_real, row16_vec, data_real);
1501  data_imag = _mm256_fmadd_pd(element_11110_vec_imag, row16_vec, data_imag);
1502 
1503  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
1504  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
1505  final_vec = _mm256_hadd_pd(final_vec, final_vec);
1506  __m128d low128 = _mm256_castpd256_pd128(final_vec);
1507  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
1508  results[mult_idx].real = _mm_cvtsd_f64(low128);
1509  results[mult_idx].imag = _mm_cvtsd_f64(high128);
1510  }
1511  input[current_idx_outer_loc] = results[0];
1512  input[current_idx_inner_loc] = results[1];
1513  input[current_idx_middle1_loc] = results[2];
1514  input[current_idx_middle1_inner_loc] = results[3];
1515  input[current_idx_middle2_loc] = results[4];
1516  input[current_idx_middle2_inner_loc] = results[5];
1517  input[current_idx_middle12_loc] = results[6];
1518  input[current_idx_middle12_inner_loc] = results[7];
1519  input[current_idx_middle3_loc] = results[8];
1520  input[current_idx_middle3_inner_loc] = results[9];
1521  input[current_idx_middle13_loc] = results[10];
1522  input[current_idx_middle13_inner_loc] = results[11];
1523  input[current_idx_middle23_loc] = results[12];
1524  input[current_idx_middle23_inner_loc] = results[13];
1525  input[current_idx_middle123_loc] = results[14];
1526  input[current_idx_middle123_inner_loc] = results[15];
1527  input[current_idx_outer_pair_loc] = results[16];
1528  input[current_idx_inner_pair_loc] = results[17];
1529  input[current_idx_middle1_pair_loc] = results[18];
1530  input[current_idx_middle1_inner_pair_loc] = results[19];
1531  input[current_idx_middle2_pair_loc] = results[20];
1532  input[current_idx_middle2_inner_pair_loc] = results[21];
1533  input[current_idx_middle12_pair_loc] = results[22];
1534  input[current_idx_middle12_inner_pair_loc] = results[23];
1535  input[current_idx_middle3_pair_loc] = results[24];
1536  input[current_idx_middle3_inner_pair_loc] = results[25];
1537  input[current_idx_middle13_pair_loc] = results[26];
1538  input[current_idx_middle13_inner_pair_loc] = results[27];
1539  input[current_idx_middle23_pair_loc] = results[28];
1540  input[current_idx_middle23_inner_pair_loc] = results[29];
1541  input[current_idx_middle123_pair_loc] = results[30];
1542  input[current_idx_middle123_inner_pair_loc] = results[31];
1543  }
1544 }
1545 
1553 void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix& two_qbit_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
1554  int inner_qbit = involved_qbits[0];
1555  int outer_qbit = involved_qbits[1];
1556  int index_step_outer = 1 << outer_qbit;
1557  int index_step_inner = 1 << inner_qbit;
1558 
1559  int num_qubits = (int)std::log2(matrix_size);
1560  std::vector<int> is_target(num_qubits, 0);
1561  for (int q : involved_qbits) is_target[q] = 1;
1562 
1563  std::vector<int> non_targets;
1564  non_targets.reserve(num_qubits - 2);
1565  for (int q = 0; q < num_qubits; ++q) {
1566  if (!is_target[q]) non_targets.push_back(q);
1567  }
1568 
1569 if (inner_qbit==0){
1570  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
1571  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
1572  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
1573  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
1574  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
1575  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
1576  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
1577  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
1578  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
1579  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
1580  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
1581  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
1582  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
1583  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
1584  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
1585  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
1586 
1587  #ifdef _WIN32
1588  #pragma omp parallel for schedule(static)
1589  #else
1590  #pragma omp parallel for simd schedule(static)
1591  #endif
1592  for (int block_idx = 0; block_idx < matrix_size>>2; block_idx++) {
1593  // Calculate base index properly
1594  int base = 0;
1595  for (size_t i = 0; i < non_targets.size(); ++i) {
1596  if (block_idx & (1ULL << i)) {
1597  base |= (1 << non_targets[i]);
1598  }
1599  }
1600 
1601  int current_idx_outer_loc = base ;
1602  int current_idx_outer_pair_loc = base | index_step_outer;
1603 
1604  // Load two consecutive complex numbers at each base location
1605  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
1606  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
1607 
1608  // Load 4 doubles = 2 complex numbers at each location
1609  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
1610  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
1611 
1612  // Compute the four matrix-vector products (one for each output element)
1613 
1614  // Result for current_idx_outer_loc (row 0 of matrix)
1615  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
1616  __m256d data_u1 = _mm256_mul_pd(element_outer_vec, mv01);
1617  __m256d data_u3 = _mm256_mul_pd(element_outer_pair_vec, mv20);
1618  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv21);
1619  __m256d data_u5 = _mm256_add_pd(data_u3, data_u0);
1620  __m256d data_u2 = _mm256_add_pd(data_u1, data_u4);
1621  __m256d data_u7 = _mm256_hadd_pd(data_u5, data_u2);
1622  __m256d data_u8 = _mm256_permute4x64_pd(data_u7, 0b11011000);
1623  __m256d data_u6 = _mm256_hadd_pd(data_u8, data_u8);
1624 
1625  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
1626  __m256d data_d1 = _mm256_mul_pd(element_outer_vec, mv41);
1627  __m256d data_d3 = _mm256_mul_pd(element_outer_pair_vec, mv60);
1628  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv61);
1629  __m256d data_d5 = _mm256_add_pd(data_d3, data_d0);
1630  __m256d data_d6 = _mm256_add_pd(data_d1, data_d4);
1631  data_d6 = _mm256_hadd_pd(data_d5, data_d6);
1632  data_d6 = _mm256_permute4x64_pd(data_d6, 0b11011000);
1633  data_d6 = _mm256_hadd_pd(data_d6, data_d6);
1634 
1635  // Result for row 2 of matrix
1636  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
1637  __m256d data_e1 = _mm256_mul_pd(element_outer_vec, mv81);
1638  __m256d data_e3 = _mm256_mul_pd(element_outer_pair_vec, mv100);
1639  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv101);
1640  __m256d data_e5 = _mm256_add_pd(data_e3, data_e0);
1641  __m256d data_e6 = _mm256_add_pd(data_e1, data_e4);
1642  data_e6 = _mm256_hadd_pd(data_e5, data_e6);
1643  data_e6 = _mm256_permute4x64_pd(data_e6, 0b11011000);
1644  data_e6 = _mm256_hadd_pd(data_e6, data_e6);
1645 
1646  // Result for row 3 of matrix
1647  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
1648  __m256d data_f1 = _mm256_mul_pd(element_outer_vec, mv121);
1649  __m256d data_f3 = _mm256_mul_pd(element_outer_pair_vec, mv140);
1650  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv141);
1651  __m256d data_f5 = _mm256_add_pd(data_f3, data_f0);
1652  __m256d data_f6 = _mm256_add_pd(data_f1, data_f4);
1653  data_f6 = _mm256_hadd_pd(data_f5, data_f6);
1654  data_f6 = _mm256_permute4x64_pd(data_f6, 0b11011000);
1655  data_f6 = _mm256_hadd_pd(data_f6, data_f6);
1656 
1657  // Store results back to the same locations where they were loaded
1658  __m128d low128u = _mm256_castpd256_pd128(data_u6);
1659  __m128d high128u = _mm256_extractf128_pd(data_u6, 1);
1660 
1661  input[current_idx_outer_loc].real = _mm_cvtsd_f64(low128u);
1662  input[current_idx_outer_loc].imag = _mm_cvtsd_f64(high128u);
1663 
1664  __m128d low128d = _mm256_castpd256_pd128(data_d6);
1665  __m128d high128d = _mm256_extractf128_pd(data_d6, 1);
1666  input[current_idx_outer_loc + 1].real = _mm_cvtsd_f64(low128d);
1667  input[current_idx_outer_loc + 1].imag = _mm_cvtsd_f64(high128d);
1668 
1669  __m128d low128e = _mm256_castpd256_pd128(data_e6);
1670  __m128d high128e = _mm256_extractf128_pd(data_e6, 1);
1671  input[current_idx_outer_pair_loc].real = _mm_cvtsd_f64(low128e);
1672  input[current_idx_outer_pair_loc].imag = _mm_cvtsd_f64(high128e);
1673 
1674  __m128d low128f = _mm256_castpd256_pd128(data_f6);
1675  __m128d high128f = _mm256_extractf128_pd(data_f6, 1);
1676  input[current_idx_outer_pair_loc + 1].real = _mm_cvtsd_f64(low128f);
1677  input[current_idx_outer_pair_loc + 1].imag = _mm_cvtsd_f64(high128f);
1678 
1679 }
1680 }
1681  else{
1682  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[0].imag, two_qbit_unitary[0].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
1683  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[0].real, two_qbit_unitary[0].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
1684  __m256d mv10 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[1].imag, two_qbit_unitary[1].real);
1685  __m256d mv11 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[1].real, two_qbit_unitary[1].imag);
1686  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[2].imag, two_qbit_unitary[2].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
1687  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[2].real, two_qbit_unitary[2].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
1688  __m256d mv30 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[3].imag, two_qbit_unitary[3].real);
1689  __m256d mv31 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[3].real, two_qbit_unitary[3].imag);
1690  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[4].imag, two_qbit_unitary[4].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
1691  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[4].real, two_qbit_unitary[4].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
1692  __m256d mv50 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[5].imag, two_qbit_unitary[5].real);
1693  __m256d mv51 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[5].real, two_qbit_unitary[5].imag);
1694  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[6].imag, two_qbit_unitary[6].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
1695  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[6].real, two_qbit_unitary[6].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
1696  __m256d mv70 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[7].imag, two_qbit_unitary[7].real);
1697  __m256d mv71 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[7].real, two_qbit_unitary[7].imag);
1698  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[8].imag, two_qbit_unitary[8].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
1699  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[8].real, two_qbit_unitary[8].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
1700  __m256d mv90 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[9].imag, two_qbit_unitary[9].real);
1701  __m256d mv91 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[9].real, two_qbit_unitary[9].imag);
1702  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[10].imag, two_qbit_unitary[10].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
1703  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[10].real, two_qbit_unitary[10].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
1704  __m256d mv110 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[11].imag, two_qbit_unitary[11].real);
1705  __m256d mv111 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[11].real, two_qbit_unitary[11].imag);
1706  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[12].imag, two_qbit_unitary[12].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
1707  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[12].real, two_qbit_unitary[12].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
1708  __m256d mv130 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[13].imag, two_qbit_unitary[13].real);
1709  __m256d mv131 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[13].real, two_qbit_unitary[13].imag);
1710  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[14].imag, two_qbit_unitary[14].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
1711  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[14].real, two_qbit_unitary[14].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
1712  __m256d mv150 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[15].imag, two_qbit_unitary[15].real);
1713  __m256d mv151 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[15].real, two_qbit_unitary[15].imag);
1714 
1715  #ifdef _WIN32
1716  #pragma omp parallel for schedule(static)
1717  #else
1718  #pragma omp parallel for simd schedule(static)
1719  #endif
1720  for (int block_idx = 0; block_idx < matrix_size>>2; block_idx+=2) {
1721  // Calculate base index properly
1722  int base = 0;
1723  for (size_t i = 0; i < non_targets.size(); ++i) {
1724  if (block_idx & (1ULL << i)) {
1725  base |= (1 << non_targets[i]);
1726  }
1727  }
1728 
1729 
1730  double* element_outer = (double*)input.get_data() + 2 * base;
1731  double* element_inner = (double*)input.get_data() + 2 * ( base | index_step_inner);
1732 
1733  double* element_outer_pair = (double*)input.get_data() + 2 * (base | index_step_outer);
1734  double* element_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_outer);
1735 
1736 
1737  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
1738  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
1739 
1740  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
1741  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
1742 
1743  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
1744  __m256d data_u1 = _mm256_mul_pd(element_inner_vec, mv10);
1745  __m256d data_u2 = _mm256_mul_pd(element_outer_vec, mv01);
1746  __m256d data_u3 = _mm256_mul_pd(element_inner_vec, mv11);
1747  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv20);
1748  __m256d data_u5 = _mm256_mul_pd(element_inner_pair_vec, mv30);
1749  __m256d data_u6 = _mm256_mul_pd(element_outer_pair_vec, mv21);
1750  __m256d data_u7 = _mm256_mul_pd(element_inner_pair_vec, mv31);
1751  __m256d data_u8 = _mm256_hadd_pd(data_u0, data_u2);
1752  __m256d data_u9 = _mm256_hadd_pd(data_u1, data_u3);
1753  __m256d data_u10 = _mm256_hadd_pd(data_u4, data_u6);
1754  __m256d data_u11 = _mm256_hadd_pd(data_u5, data_u7);
1755  __m256d data_u = _mm256_add_pd(data_u8, data_u9);
1756  data_u = _mm256_add_pd(data_u, data_u10);
1757  data_u = _mm256_add_pd(data_u, data_u11);
1758 
1759  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
1760  __m256d data_d1 = _mm256_mul_pd(element_inner_vec, mv50);
1761  __m256d data_d2 = _mm256_mul_pd(element_outer_vec, mv41);
1762  __m256d data_d3 = _mm256_mul_pd(element_inner_vec, mv51);
1763  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv60);
1764  __m256d data_d5 = _mm256_mul_pd(element_inner_pair_vec, mv70);
1765  __m256d data_d6 = _mm256_mul_pd(element_outer_pair_vec, mv61);
1766  __m256d data_d7 = _mm256_mul_pd(element_inner_pair_vec, mv71);
1767  __m256d data_d8 = _mm256_hadd_pd(data_d0, data_d2);
1768  __m256d data_d9 = _mm256_hadd_pd(data_d1, data_d3);
1769  __m256d data_d10 = _mm256_hadd_pd(data_d4, data_d6);
1770  __m256d data_d11 = _mm256_hadd_pd(data_d5, data_d7);
1771  __m256d data_d = _mm256_add_pd(data_d8, data_d9);
1772  data_d = _mm256_add_pd(data_d, data_d10);
1773  data_d = _mm256_add_pd(data_d, data_d11);
1774 
1775  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
1776  __m256d data_e1 = _mm256_mul_pd(element_inner_vec, mv90);
1777  __m256d data_e2 = _mm256_mul_pd(element_outer_vec, mv81);
1778  __m256d data_e3 = _mm256_mul_pd(element_inner_vec, mv91);
1779  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv100);
1780  __m256d data_e5 = _mm256_mul_pd(element_inner_pair_vec, mv110);
1781  __m256d data_e6 = _mm256_mul_pd(element_outer_pair_vec, mv101);
1782  __m256d data_e7 = _mm256_mul_pd(element_inner_pair_vec, mv111);
1783  __m256d data_e8 = _mm256_hadd_pd(data_e0, data_e2);
1784  __m256d data_e9 = _mm256_hadd_pd(data_e1, data_e3);
1785  __m256d data_e10 = _mm256_hadd_pd(data_e4, data_e6);
1786  __m256d data_e11 = _mm256_hadd_pd(data_e5, data_e7);
1787  __m256d data_e = _mm256_add_pd(data_e8, data_e9);
1788  data_e = _mm256_add_pd(data_e, data_e10);
1789  data_e = _mm256_add_pd(data_e, data_e11);
1790 
1791  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
1792  __m256d data_f1 = _mm256_mul_pd(element_inner_vec, mv130);
1793  __m256d data_f2 = _mm256_mul_pd(element_outer_vec, mv121);
1794  __m256d data_f3 = _mm256_mul_pd(element_inner_vec, mv131);
1795  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv140);
1796  __m256d data_f5 = _mm256_mul_pd(element_inner_pair_vec, mv150);
1797  __m256d data_f6 = _mm256_mul_pd(element_outer_pair_vec, mv141);
1798  __m256d data_f7 = _mm256_mul_pd(element_inner_pair_vec, mv151);
1799  __m256d data_f8 = _mm256_hadd_pd(data_f0, data_f2);
1800  __m256d data_f9 = _mm256_hadd_pd(data_f1, data_f3);
1801  __m256d data_f10 = _mm256_hadd_pd(data_f4, data_f6);
1802  __m256d data_f11 = _mm256_hadd_pd(data_f5, data_f7);
1803  __m256d data_f = _mm256_add_pd(data_f8, data_f9);
1804  data_f = _mm256_add_pd(data_f, data_f10);
1805  data_f = _mm256_add_pd(data_f, data_f11);
1806 
1807  _mm256_storeu_pd(element_outer, data_u);
1808  _mm256_storeu_pd(element_inner, data_d);
1809  _mm256_storeu_pd(element_outer_pair, data_e);
1810  _mm256_storeu_pd(element_inner_pair, data_f);
1811  }
1812 
1813  }
1814 
1815 
1816 
1817 }
1825 void apply_2qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix& two_qbit_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
1826  int inner_qbit = involved_qbits[0];
1827  int outer_qbit = involved_qbits[1];
1828  int index_step_outer = 1 << outer_qbit;
1829  int index_step_inner = 1 << inner_qbit;
1830 
1831  int num_qubits = (int)std::log2(matrix_size);
1832  std::vector<int> is_target(num_qubits, 0);
1833  for (int q : involved_qbits) is_target[q] = 1;
1834 
1835  std::vector<int> non_targets;
1836  non_targets.reserve(num_qubits - 2);
1837  for (int q = 0; q < num_qubits; ++q) {
1838  if (!is_target[q]) non_targets.push_back(q);
1839  }
1840 
1841  if (inner_qbit == 0) {
1842  // Pre-compute all matrix vectors outside the parallel region
1843  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
1844  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
1845  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
1846  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
1847  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
1848  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
1849  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
1850  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
1851  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
1852  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
1853  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
1854  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
1855  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
1856  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
1857  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
1858  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
1859 
1860  // TBB parallel_for with blocked_range
1861  tbb::parallel_for(
1862  tbb::blocked_range<int>(0, matrix_size >> 2),
1863  [&](const tbb::blocked_range<int>& range) {
1864  for (int block_idx = range.begin(); block_idx != range.end(); ++block_idx) {
1865  // Calculate base index properly
1866  int base = 0;
1867  for (size_t i = 0; i < non_targets.size(); ++i) {
1868  if (block_idx & (1ULL << i)) {
1869  base |= (1 << non_targets[i]);
1870  }
1871  }
1872 
1873  int current_idx_outer_loc = base;
1874  int current_idx_outer_pair_loc = base | index_step_outer;
1875 
1876  // Load two consecutive complex numbers at each base location
1877  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
1878  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
1879 
1880  // Load 4 doubles = 2 complex numbers at each location
1881  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
1882  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
1883 
1884  // Compute the four matrix-vector products (one for each output element)
1885 
1886  // Result for current_idx_outer_loc (row 0 of matrix)
1887  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
1888  __m256d data_u1 = _mm256_mul_pd(element_outer_vec, mv01);
1889  __m256d data_u3 = _mm256_mul_pd(element_outer_pair_vec, mv20);
1890  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv21);
1891  __m256d data_u5 = _mm256_add_pd(data_u3, data_u0);
1892  __m256d data_u2 = _mm256_add_pd(data_u1, data_u4);
1893  __m256d data_u7 = _mm256_hadd_pd(data_u5, data_u2);
1894  __m256d data_u8 = _mm256_permute4x64_pd(data_u7, 0b11011000);
1895  __m256d data_u6 = _mm256_hadd_pd(data_u8, data_u8);
1896 
1897  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
1898  __m256d data_d1 = _mm256_mul_pd(element_outer_vec, mv41);
1899  __m256d data_d3 = _mm256_mul_pd(element_outer_pair_vec, mv60);
1900  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv61);
1901  __m256d data_d5 = _mm256_add_pd(data_d3, data_d0);
1902  __m256d data_d6 = _mm256_add_pd(data_d1, data_d4);
1903  data_d6 = _mm256_hadd_pd(data_d5, data_d6);
1904  data_d6 = _mm256_permute4x64_pd(data_d6, 0b11011000);
1905  data_d6 = _mm256_hadd_pd(data_d6, data_d6);
1906 
1907  // Result for row 2 of matrix
1908  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
1909  __m256d data_e1 = _mm256_mul_pd(element_outer_vec, mv81);
1910  __m256d data_e3 = _mm256_mul_pd(element_outer_pair_vec, mv100);
1911  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv101);
1912  __m256d data_e5 = _mm256_add_pd(data_e3, data_e0);
1913  __m256d data_e6 = _mm256_add_pd(data_e1, data_e4);
1914  data_e6 = _mm256_hadd_pd(data_e5, data_e6);
1915  data_e6 = _mm256_permute4x64_pd(data_e6, 0b11011000);
1916  data_e6 = _mm256_hadd_pd(data_e6, data_e6);
1917 
1918  // Result for row 3 of matrix
1919  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
1920  __m256d data_f1 = _mm256_mul_pd(element_outer_vec, mv121);
1921  __m256d data_f3 = _mm256_mul_pd(element_outer_pair_vec, mv140);
1922  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv141);
1923  __m256d data_f5 = _mm256_add_pd(data_f3, data_f0);
1924  __m256d data_f6 = _mm256_add_pd(data_f1, data_f4);
1925  data_f6 = _mm256_hadd_pd(data_f5, data_f6);
1926  data_f6 = _mm256_permute4x64_pd(data_f6, 0b11011000);
1927  data_f6 = _mm256_hadd_pd(data_f6, data_f6);
1928 
1929  // Store results back to the same locations where they were loaded
1930  __m128d low128u = _mm256_castpd256_pd128(data_u6);
1931  __m128d high128u = _mm256_extractf128_pd(data_u6, 1);
1932 
1933  input[current_idx_outer_loc].real = _mm_cvtsd_f64(low128u);
1934  input[current_idx_outer_loc].imag = _mm_cvtsd_f64(high128u);
1935 
1936  __m128d low128d = _mm256_castpd256_pd128(data_d6);
1937  __m128d high128d = _mm256_extractf128_pd(data_d6, 1);
1938  input[current_idx_outer_loc + 1].real = _mm_cvtsd_f64(low128d);
1939  input[current_idx_outer_loc + 1].imag = _mm_cvtsd_f64(high128d);
1940 
1941  __m128d low128e = _mm256_castpd256_pd128(data_e6);
1942  __m128d high128e = _mm256_extractf128_pd(data_e6, 1);
1943  input[current_idx_outer_pair_loc].real = _mm_cvtsd_f64(low128e);
1944  input[current_idx_outer_pair_loc].imag = _mm_cvtsd_f64(high128e);
1945 
1946  __m128d low128f = _mm256_castpd256_pd128(data_f6);
1947  __m128d high128f = _mm256_extractf128_pd(data_f6, 1);
1948  input[current_idx_outer_pair_loc + 1].real = _mm_cvtsd_f64(low128f);
1949  input[current_idx_outer_pair_loc + 1].imag = _mm_cvtsd_f64(high128f);
1950  }
1951  }
1952  );
1953  }
1954  else {
1955  // Pre-compute all matrix vectors outside the parallel region
1956  __m256d mv00 = _mm256_set_pd(-two_qbit_unitary[0].imag, two_qbit_unitary[0].real, -two_qbit_unitary[0].imag, two_qbit_unitary[0].real);
1957  __m256d mv01 = _mm256_set_pd( two_qbit_unitary[0].real, two_qbit_unitary[0].imag, two_qbit_unitary[0].real, two_qbit_unitary[0].imag);
1958  __m256d mv10 = _mm256_set_pd(-two_qbit_unitary[1].imag, two_qbit_unitary[1].real, -two_qbit_unitary[1].imag, two_qbit_unitary[1].real);
1959  __m256d mv11 = _mm256_set_pd( two_qbit_unitary[1].real, two_qbit_unitary[1].imag, two_qbit_unitary[1].real, two_qbit_unitary[1].imag);
1960  __m256d mv20 = _mm256_set_pd(-two_qbit_unitary[2].imag, two_qbit_unitary[2].real, -two_qbit_unitary[2].imag, two_qbit_unitary[2].real);
1961  __m256d mv21 = _mm256_set_pd( two_qbit_unitary[2].real, two_qbit_unitary[2].imag, two_qbit_unitary[2].real, two_qbit_unitary[2].imag);
1962  __m256d mv30 = _mm256_set_pd(-two_qbit_unitary[3].imag, two_qbit_unitary[3].real, -two_qbit_unitary[3].imag, two_qbit_unitary[3].real);
1963  __m256d mv31 = _mm256_set_pd( two_qbit_unitary[3].real, two_qbit_unitary[3].imag, two_qbit_unitary[3].real, two_qbit_unitary[3].imag);
1964  __m256d mv40 = _mm256_set_pd(-two_qbit_unitary[4].imag, two_qbit_unitary[4].real, -two_qbit_unitary[4].imag, two_qbit_unitary[4].real);
1965  __m256d mv41 = _mm256_set_pd( two_qbit_unitary[4].real, two_qbit_unitary[4].imag, two_qbit_unitary[4].real, two_qbit_unitary[4].imag);
1966  __m256d mv50 = _mm256_set_pd(-two_qbit_unitary[5].imag, two_qbit_unitary[5].real, -two_qbit_unitary[5].imag, two_qbit_unitary[5].real);
1967  __m256d mv51 = _mm256_set_pd( two_qbit_unitary[5].real, two_qbit_unitary[5].imag, two_qbit_unitary[5].real, two_qbit_unitary[5].imag);
1968  __m256d mv60 = _mm256_set_pd(-two_qbit_unitary[6].imag, two_qbit_unitary[6].real, -two_qbit_unitary[6].imag, two_qbit_unitary[6].real);
1969  __m256d mv61 = _mm256_set_pd( two_qbit_unitary[6].real, two_qbit_unitary[6].imag, two_qbit_unitary[6].real, two_qbit_unitary[6].imag);
1970  __m256d mv70 = _mm256_set_pd(-two_qbit_unitary[7].imag, two_qbit_unitary[7].real, -two_qbit_unitary[7].imag, two_qbit_unitary[7].real);
1971  __m256d mv71 = _mm256_set_pd( two_qbit_unitary[7].real, two_qbit_unitary[7].imag, two_qbit_unitary[7].real, two_qbit_unitary[7].imag);
1972  __m256d mv80 = _mm256_set_pd(-two_qbit_unitary[8].imag, two_qbit_unitary[8].real, -two_qbit_unitary[8].imag, two_qbit_unitary[8].real);
1973  __m256d mv81 = _mm256_set_pd( two_qbit_unitary[8].real, two_qbit_unitary[8].imag, two_qbit_unitary[8].real, two_qbit_unitary[8].imag);
1974  __m256d mv90 = _mm256_set_pd(-two_qbit_unitary[9].imag, two_qbit_unitary[9].real, -two_qbit_unitary[9].imag, two_qbit_unitary[9].real);
1975  __m256d mv91 = _mm256_set_pd( two_qbit_unitary[9].real, two_qbit_unitary[9].imag, two_qbit_unitary[9].real, two_qbit_unitary[9].imag);
1976  __m256d mv100 = _mm256_set_pd(-two_qbit_unitary[10].imag, two_qbit_unitary[10].real, -two_qbit_unitary[10].imag, two_qbit_unitary[10].real);
1977  __m256d mv101 = _mm256_set_pd( two_qbit_unitary[10].real, two_qbit_unitary[10].imag, two_qbit_unitary[10].real, two_qbit_unitary[10].imag);
1978  __m256d mv110 = _mm256_set_pd(-two_qbit_unitary[11].imag, two_qbit_unitary[11].real, -two_qbit_unitary[11].imag, two_qbit_unitary[11].real);
1979  __m256d mv111 = _mm256_set_pd( two_qbit_unitary[11].real, two_qbit_unitary[11].imag, two_qbit_unitary[11].real, two_qbit_unitary[11].imag);
1980  __m256d mv120 = _mm256_set_pd(-two_qbit_unitary[12].imag, two_qbit_unitary[12].real, -two_qbit_unitary[12].imag, two_qbit_unitary[12].real);
1981  __m256d mv121 = _mm256_set_pd( two_qbit_unitary[12].real, two_qbit_unitary[12].imag, two_qbit_unitary[12].real, two_qbit_unitary[12].imag);
1982  __m256d mv130 = _mm256_set_pd(-two_qbit_unitary[13].imag, two_qbit_unitary[13].real, -two_qbit_unitary[13].imag, two_qbit_unitary[13].real);
1983  __m256d mv131 = _mm256_set_pd( two_qbit_unitary[13].real, two_qbit_unitary[13].imag, two_qbit_unitary[13].real, two_qbit_unitary[13].imag);
1984  __m256d mv140 = _mm256_set_pd(-two_qbit_unitary[14].imag, two_qbit_unitary[14].real, -two_qbit_unitary[14].imag, two_qbit_unitary[14].real);
1985  __m256d mv141 = _mm256_set_pd( two_qbit_unitary[14].real, two_qbit_unitary[14].imag, two_qbit_unitary[14].real, two_qbit_unitary[14].imag);
1986  __m256d mv150 = _mm256_set_pd(-two_qbit_unitary[15].imag, two_qbit_unitary[15].real, -two_qbit_unitary[15].imag, two_qbit_unitary[15].real);
1987  __m256d mv151 = _mm256_set_pd( two_qbit_unitary[15].real, two_qbit_unitary[15].imag, two_qbit_unitary[15].real, two_qbit_unitary[15].imag);
1988 
1989  // TBB parallel_for with blocked_range - note the step size of 2
1990  tbb::parallel_for(
1991  tbb::blocked_range<int>(0, matrix_size >> 2, 2), // grain_size of 2 to match block_idx+=2
1992  [&](const tbb::blocked_range<int>& range) {
1993  for (int block_idx = range.begin(); block_idx < range.end(); block_idx += 2) {
1994  // Calculate base index properly
1995  int base = 0;
1996  for (size_t i = 0; i < non_targets.size(); ++i) {
1997  if (block_idx & (1ULL << i)) {
1998  base |= (1 << non_targets[i]);
1999  }
2000  }
2001 
2002  double* element_outer = (double*)input.get_data() + 2 * base;
2003  double* element_inner = (double*)input.get_data() + 2 * ( base | index_step_inner);
2004 
2005  double* element_outer_pair = (double*)input.get_data() + 2 * (base | index_step_outer);
2006  double* element_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_outer);
2007 
2008 
2009  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
2010  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
2011 
2012  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
2013  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
2014 
2015  __m256d data_u0 = _mm256_mul_pd(element_outer_vec, mv00);
2016  __m256d data_u1 = _mm256_mul_pd(element_inner_vec, mv10);
2017  __m256d data_u2 = _mm256_mul_pd(element_outer_vec, mv01);
2018  __m256d data_u3 = _mm256_mul_pd(element_inner_vec, mv11);
2019  __m256d data_u4 = _mm256_mul_pd(element_outer_pair_vec, mv20);
2020  __m256d data_u5 = _mm256_mul_pd(element_inner_pair_vec, mv30);
2021  __m256d data_u6 = _mm256_mul_pd(element_outer_pair_vec, mv21);
2022  __m256d data_u7 = _mm256_mul_pd(element_inner_pair_vec, mv31);
2023  __m256d data_u8 = _mm256_hadd_pd(data_u0, data_u2);
2024  __m256d data_u9 = _mm256_hadd_pd(data_u1, data_u3);
2025  __m256d data_u10 = _mm256_hadd_pd(data_u4, data_u6);
2026  __m256d data_u11 = _mm256_hadd_pd(data_u5, data_u7);
2027  __m256d data_u = _mm256_add_pd(data_u8, data_u9);
2028  data_u = _mm256_add_pd(data_u, data_u10);
2029  data_u = _mm256_add_pd(data_u, data_u11);
2030 
2031  __m256d data_d0 = _mm256_mul_pd(element_outer_vec, mv40);
2032  __m256d data_d1 = _mm256_mul_pd(element_inner_vec, mv50);
2033  __m256d data_d2 = _mm256_mul_pd(element_outer_vec, mv41);
2034  __m256d data_d3 = _mm256_mul_pd(element_inner_vec, mv51);
2035  __m256d data_d4 = _mm256_mul_pd(element_outer_pair_vec, mv60);
2036  __m256d data_d5 = _mm256_mul_pd(element_inner_pair_vec, mv70);
2037  __m256d data_d6 = _mm256_mul_pd(element_outer_pair_vec, mv61);
2038  __m256d data_d7 = _mm256_mul_pd(element_inner_pair_vec, mv71);
2039  __m256d data_d8 = _mm256_hadd_pd(data_d0, data_d2);
2040  __m256d data_d9 = _mm256_hadd_pd(data_d1, data_d3);
2041  __m256d data_d10 = _mm256_hadd_pd(data_d4, data_d6);
2042  __m256d data_d11 = _mm256_hadd_pd(data_d5, data_d7);
2043  __m256d data_d = _mm256_add_pd(data_d8, data_d9);
2044  data_d = _mm256_add_pd(data_d, data_d10);
2045  data_d = _mm256_add_pd(data_d, data_d11);
2046 
2047  __m256d data_e0 = _mm256_mul_pd(element_outer_vec, mv80);
2048  __m256d data_e1 = _mm256_mul_pd(element_inner_vec, mv90);
2049  __m256d data_e2 = _mm256_mul_pd(element_outer_vec, mv81);
2050  __m256d data_e3 = _mm256_mul_pd(element_inner_vec, mv91);
2051  __m256d data_e4 = _mm256_mul_pd(element_outer_pair_vec, mv100);
2052  __m256d data_e5 = _mm256_mul_pd(element_inner_pair_vec, mv110);
2053  __m256d data_e6 = _mm256_mul_pd(element_outer_pair_vec, mv101);
2054  __m256d data_e7 = _mm256_mul_pd(element_inner_pair_vec, mv111);
2055  __m256d data_e8 = _mm256_hadd_pd(data_e0, data_e2);
2056  __m256d data_e9 = _mm256_hadd_pd(data_e1, data_e3);
2057  __m256d data_e10 = _mm256_hadd_pd(data_e4, data_e6);
2058  __m256d data_e11 = _mm256_hadd_pd(data_e5, data_e7);
2059  __m256d data_e = _mm256_add_pd(data_e8, data_e9);
2060  data_e = _mm256_add_pd(data_e, data_e10);
2061  data_e = _mm256_add_pd(data_e, data_e11);
2062 
2063  __m256d data_f0 = _mm256_mul_pd(element_outer_vec, mv120);
2064  __m256d data_f1 = _mm256_mul_pd(element_inner_vec, mv130);
2065  __m256d data_f2 = _mm256_mul_pd(element_outer_vec, mv121);
2066  __m256d data_f3 = _mm256_mul_pd(element_inner_vec, mv131);
2067  __m256d data_f4 = _mm256_mul_pd(element_outer_pair_vec, mv140);
2068  __m256d data_f5 = _mm256_mul_pd(element_inner_pair_vec, mv150);
2069  __m256d data_f6 = _mm256_mul_pd(element_outer_pair_vec, mv141);
2070  __m256d data_f7 = _mm256_mul_pd(element_inner_pair_vec, mv151);
2071  __m256d data_f8 = _mm256_hadd_pd(data_f0, data_f2);
2072  __m256d data_f9 = _mm256_hadd_pd(data_f1, data_f3);
2073  __m256d data_f10 = _mm256_hadd_pd(data_f4, data_f6);
2074  __m256d data_f11 = _mm256_hadd_pd(data_f5, data_f7);
2075  __m256d data_f = _mm256_add_pd(data_f8, data_f9);
2076  data_f = _mm256_add_pd(data_f, data_f10);
2077  data_f = _mm256_add_pd(data_f, data_f11);
2078 
2079  _mm256_storeu_pd(element_outer, data_u);
2080  _mm256_storeu_pd(element_inner, data_d);
2081  _mm256_storeu_pd(element_outer_pair, data_e);
2082  _mm256_storeu_pd(element_inner_pair, data_f);
2083  }
2084  }
2085  );
2086  }
2087 }
2095 void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
2096  int inner_qbit = involved_qbits[0];
2097  int middle_qbit = involved_qbits[1];
2098  int outer_qbit = involved_qbits[2];
2099 
2100  int index_step_inner = 1 << inner_qbit;
2101  int index_step_middle = 1 << middle_qbit;
2102  int index_step_outer = 1 << outer_qbit;
2103 
2104  // Use the same approach as n-qubit for all cases
2105  int qubit_num = (int) std::log2(input.rows);
2106 
2107  // Identify non-involved qubits
2108  std::vector<int> is_target(qubit_num, 0);
2109  for (int q : involved_qbits) is_target[q] = 1;
2110 
2111  std::vector<int> non_targets;
2112  non_targets.reserve(qubit_num - 3);
2113  for (int q = 0; q < qubit_num; ++q) {
2114  if (!is_target[q]) non_targets.push_back(q);
2115  }
2116 
2117  if (inner_qbit == 0) {
2118  // Consecutive case - create packed vectors
2119  CREATE_MATRIX_VECTOR_CONSECUTIVE(0) CREATE_MATRIX_VECTOR_CONSECUTIVE(2)
2120  CREATE_MATRIX_VECTOR_CONSECUTIVE(4) CREATE_MATRIX_VECTOR_CONSECUTIVE(6)
2121  CREATE_MATRIX_VECTOR_CONSECUTIVE(8) CREATE_MATRIX_VECTOR_CONSECUTIVE(10)
2122  CREATE_MATRIX_VECTOR_CONSECUTIVE(12) CREATE_MATRIX_VECTOR_CONSECUTIVE(14)
2123  CREATE_MATRIX_VECTOR_CONSECUTIVE(16) CREATE_MATRIX_VECTOR_CONSECUTIVE(18)
2124  CREATE_MATRIX_VECTOR_CONSECUTIVE(20) CREATE_MATRIX_VECTOR_CONSECUTIVE(22)
2125  CREATE_MATRIX_VECTOR_CONSECUTIVE(24) CREATE_MATRIX_VECTOR_CONSECUTIVE(26)
2126  CREATE_MATRIX_VECTOR_CONSECUTIVE(28) CREATE_MATRIX_VECTOR_CONSECUTIVE(30)
2127  CREATE_MATRIX_VECTOR_CONSECUTIVE(32) CREATE_MATRIX_VECTOR_CONSECUTIVE(34)
2128  CREATE_MATRIX_VECTOR_CONSECUTIVE(36) CREATE_MATRIX_VECTOR_CONSECUTIVE(38)
2129  CREATE_MATRIX_VECTOR_CONSECUTIVE(40) CREATE_MATRIX_VECTOR_CONSECUTIVE(42)
2130  CREATE_MATRIX_VECTOR_CONSECUTIVE(44) CREATE_MATRIX_VECTOR_CONSECUTIVE(46)
2131  CREATE_MATRIX_VECTOR_CONSECUTIVE(48) CREATE_MATRIX_VECTOR_CONSECUTIVE(50)
2132  CREATE_MATRIX_VECTOR_CONSECUTIVE(52) CREATE_MATRIX_VECTOR_CONSECUTIVE(54)
2133  CREATE_MATRIX_VECTOR_CONSECUTIVE(56) CREATE_MATRIX_VECTOR_CONSECUTIVE(58)
2134  CREATE_MATRIX_VECTOR_CONSECUTIVE(60) CREATE_MATRIX_VECTOR_CONSECUTIVE(62)
2135 
2136  #ifdef _WIN32
2137  #pragma omp parallel for schedule(static)
2138  #else
2139  #pragma omp parallel for simd schedule(static)
2140  #endif
2141  for (int iter_idx = 0; iter_idx < matrix_size>>3; iter_idx++) {
2142  int base = 0;
2143  for (size_t i = 0; i < non_targets.size(); ++i) {
2144  if (iter_idx & (1ULL << i)) {
2145  base |= (1 << non_targets[i]);
2146  }
2147  }
2148 
2149  double* ptr_000 = (double*)input.get_data() + 2 * base;
2150  double* ptr_010 = (double*)input.get_data() + 2 * (base | index_step_middle);
2151  double* ptr_100 = (double*)input.get_data() + 2 * (base | index_step_outer);
2152  double* ptr_110 = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
2153 
2154  __m256d element_000_vec = _mm256_loadu_pd(ptr_000); // Loads indices[0] and [1]
2155  __m256d element_010_vec = _mm256_loadu_pd(ptr_010); // Loads indices[2] and [3]
2156  __m256d element_100_vec = _mm256_loadu_pd(ptr_100); // Loads indices[4] and [5]
2157  __m256d element_110_vec = _mm256_loadu_pd(ptr_110); // Loads indices[6] and [7]
2158 
2159  QGD_Complex16 results[8];
2160 
2161  int row_idx = 0;
2162  COMPUTE_3QBIT_ROW_CONSECUTIVE(u, 0, 2, 4, 6)
2163  row_idx = 1;
2164  COMPUTE_3QBIT_ROW_CONSECUTIVE(d, 8, 10, 12, 14)
2165  row_idx = 2;
2166  COMPUTE_3QBIT_ROW_CONSECUTIVE(e, 16, 18, 20, 22)
2167  row_idx = 3;
2168  COMPUTE_3QBIT_ROW_CONSECUTIVE(f, 24, 26, 28, 30)
2169  row_idx = 4;
2170  COMPUTE_3QBIT_ROW_CONSECUTIVE(g, 32, 34, 36, 38)
2171  row_idx = 5;
2172  COMPUTE_3QBIT_ROW_CONSECUTIVE(h, 40, 42, 44, 46)
2173  row_idx = 6;
2174  COMPUTE_3QBIT_ROW_CONSECUTIVE(i, 48, 50, 52, 54)
2175  row_idx = 7;
2176  COMPUTE_3QBIT_ROW_CONSECUTIVE(j, 56, 58, 60, 62)
2177 
2178  // Store results at the correct indices
2179  input[base] = results[0];
2180  input[base+1] = results[1];
2181  input[(base | index_step_middle)] = results[2];
2182  input[(base | index_step_middle)+1] = results[3];
2183  input[(base | index_step_outer)] = results[4];
2184  input[(base | index_step_outer)+1] = results[5];
2185  input[(base | index_step_middle | index_step_outer)] = results[6];
2186  input[(base | index_step_middle | index_step_outer)+1] = results[7];
2187  }
2188  }
2189  else {
2190  // Non-consecutive case
2191  CREATE_MATRIX_VECTOR(0) CREATE_MATRIX_VECTOR(1) CREATE_MATRIX_VECTOR(2) CREATE_MATRIX_VECTOR(3)
2192  CREATE_MATRIX_VECTOR(4) CREATE_MATRIX_VECTOR(5) CREATE_MATRIX_VECTOR(6) CREATE_MATRIX_VECTOR(7)
2193  CREATE_MATRIX_VECTOR(8) CREATE_MATRIX_VECTOR(9) CREATE_MATRIX_VECTOR(10) CREATE_MATRIX_VECTOR(11)
2194  CREATE_MATRIX_VECTOR(12) CREATE_MATRIX_VECTOR(13) CREATE_MATRIX_VECTOR(14) CREATE_MATRIX_VECTOR(15)
2195  CREATE_MATRIX_VECTOR(16) CREATE_MATRIX_VECTOR(17) CREATE_MATRIX_VECTOR(18) CREATE_MATRIX_VECTOR(19)
2196  CREATE_MATRIX_VECTOR(20) CREATE_MATRIX_VECTOR(21) CREATE_MATRIX_VECTOR(22) CREATE_MATRIX_VECTOR(23)
2197  CREATE_MATRIX_VECTOR(24) CREATE_MATRIX_VECTOR(25) CREATE_MATRIX_VECTOR(26) CREATE_MATRIX_VECTOR(27)
2198  CREATE_MATRIX_VECTOR(28) CREATE_MATRIX_VECTOR(29) CREATE_MATRIX_VECTOR(30) CREATE_MATRIX_VECTOR(31)
2199  CREATE_MATRIX_VECTOR(32) CREATE_MATRIX_VECTOR(33) CREATE_MATRIX_VECTOR(34) CREATE_MATRIX_VECTOR(35)
2200  CREATE_MATRIX_VECTOR(36) CREATE_MATRIX_VECTOR(37) CREATE_MATRIX_VECTOR(38) CREATE_MATRIX_VECTOR(39)
2201  CREATE_MATRIX_VECTOR(40) CREATE_MATRIX_VECTOR(41) CREATE_MATRIX_VECTOR(42) CREATE_MATRIX_VECTOR(43)
2202  CREATE_MATRIX_VECTOR(44) CREATE_MATRIX_VECTOR(45) CREATE_MATRIX_VECTOR(46) CREATE_MATRIX_VECTOR(47)
2203  CREATE_MATRIX_VECTOR(48) CREATE_MATRIX_VECTOR(49) CREATE_MATRIX_VECTOR(50) CREATE_MATRIX_VECTOR(51)
2204  CREATE_MATRIX_VECTOR(52) CREATE_MATRIX_VECTOR(53) CREATE_MATRIX_VECTOR(54) CREATE_MATRIX_VECTOR(55)
2205  CREATE_MATRIX_VECTOR(56) CREATE_MATRIX_VECTOR(57) CREATE_MATRIX_VECTOR(58) CREATE_MATRIX_VECTOR(59)
2206  CREATE_MATRIX_VECTOR(60) CREATE_MATRIX_VECTOR(61) CREATE_MATRIX_VECTOR(62) CREATE_MATRIX_VECTOR(63)
2207  #ifdef _WIN32
2208  #pragma omp parallel for schedule(static)
2209  #else
2210  #pragma omp parallel for simd schedule(static)
2211  #endif
2212  for (int iter_idx = 0; iter_idx < matrix_size>>3; iter_idx+=2) {
2213 
2214  int base = 0;
2215  for (size_t i = 0; i < non_targets.size(); ++i) {
2216  if (iter_idx & (1ULL << i)) {
2217  base |= (1 << non_targets[i]);
2218  }
2219  }
2220 
2221  double* element_outer = (double*)input.get_data() + 2 * base;
2222  double* element_inner = (double*)input.get_data() + 2 * (base | index_step_inner);
2223  double* element_middle = (double*)input.get_data() + 2 * (base | index_step_middle);
2224  double* element_middle_inner = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle);
2225  double* element_outer_pair = (double*)input.get_data() + 2 * (base | index_step_outer);
2226  double* element_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_outer);
2227  double* element_middle_pair = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
2228  double* element_middle_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle | index_step_outer);
2229 
2230  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
2231  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
2232  __m256d element_middle_vec = _mm256_loadu_pd(element_middle);
2233  __m256d element_middle_inner_vec = _mm256_loadu_pd(element_middle_inner);
2234  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
2235  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
2236  __m256d element_middle_pair_vec = _mm256_loadu_pd(element_middle_pair);
2237  __m256d element_middle_inner_pair_vec = _mm256_loadu_pd(element_middle_inner_pair);
2238 
2239  // Compute all 8 rows using macros
2240  COMPUTE_3QBIT_ROW(u, 0, 1, 2, 3, 4, 5, 6, 7)
2241  COMPUTE_3QBIT_ROW(d, 8, 9, 10, 11, 12, 13, 14, 15)
2242  COMPUTE_3QBIT_ROW(e, 16, 17, 18, 19, 20, 21, 22, 23)
2243  COMPUTE_3QBIT_ROW(f, 24, 25, 26, 27, 28, 29, 30, 31)
2244  COMPUTE_3QBIT_ROW(g, 32, 33, 34, 35, 36, 37, 38, 39)
2245  COMPUTE_3QBIT_ROW(h, 40, 41, 42, 43, 44, 45, 46, 47)
2246  COMPUTE_3QBIT_ROW(i, 48, 49, 50, 51, 52, 53, 54, 55)
2247  COMPUTE_3QBIT_ROW(j, 56, 57, 58, 59, 60, 61, 62, 63)
2248 
2249  // Store results
2250  _mm256_storeu_pd(element_outer, data_u);
2251  _mm256_storeu_pd(element_inner, data_d);
2252  _mm256_storeu_pd(element_middle, data_e);
2253  _mm256_storeu_pd(element_middle_inner, data_f);
2254  _mm256_storeu_pd(element_outer_pair, data_g);
2255  _mm256_storeu_pd(element_inner_pair, data_h);
2256  _mm256_storeu_pd(element_middle_pair, data_i);
2257  _mm256_storeu_pd(element_middle_inner_pair, data_j);
2258  }
2259  }
2260 }
2268 void apply_3qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
2269  int inner_qbit = involved_qbits[0];
2270  int middle_qbit = involved_qbits[1];
2271  int outer_qbit = involved_qbits[2];
2272 
2273  int index_step_inner = 1 << inner_qbit;
2274  int index_step_middle = 1 << middle_qbit;
2275  int index_step_outer = 1 << outer_qbit;
2276 
2277  // Use the same approach as n-qubit for all cases
2278  int qubit_num = (int) std::log2(input.rows);
2279 
2280  // Identify non-involved qubits
2281  std::vector<int> is_target(qubit_num, 0);
2282  for (int q : involved_qbits) is_target[q] = 1;
2283 
2284  std::vector<int> non_targets;
2285  non_targets.reserve(qubit_num - 3);
2286  for (int q = 0; q < qubit_num; ++q) {
2287  if (!is_target[q]) non_targets.push_back(q);
2288  }
2289 
2290  if (inner_qbit == 0) {
2291  // Pre-compute all matrix vectors outside the parallel region
2292  CREATE_MATRIX_VECTOR_CONSECUTIVE(0) CREATE_MATRIX_VECTOR_CONSECUTIVE(2)
2293  CREATE_MATRIX_VECTOR_CONSECUTIVE(4) CREATE_MATRIX_VECTOR_CONSECUTIVE(6)
2294  CREATE_MATRIX_VECTOR_CONSECUTIVE(8) CREATE_MATRIX_VECTOR_CONSECUTIVE(10)
2295  CREATE_MATRIX_VECTOR_CONSECUTIVE(12) CREATE_MATRIX_VECTOR_CONSECUTIVE(14)
2296  CREATE_MATRIX_VECTOR_CONSECUTIVE(16) CREATE_MATRIX_VECTOR_CONSECUTIVE(18)
2297  CREATE_MATRIX_VECTOR_CONSECUTIVE(20) CREATE_MATRIX_VECTOR_CONSECUTIVE(22)
2298  CREATE_MATRIX_VECTOR_CONSECUTIVE(24) CREATE_MATRIX_VECTOR_CONSECUTIVE(26)
2299  CREATE_MATRIX_VECTOR_CONSECUTIVE(28) CREATE_MATRIX_VECTOR_CONSECUTIVE(30)
2300  CREATE_MATRIX_VECTOR_CONSECUTIVE(32) CREATE_MATRIX_VECTOR_CONSECUTIVE(34)
2301  CREATE_MATRIX_VECTOR_CONSECUTIVE(36) CREATE_MATRIX_VECTOR_CONSECUTIVE(38)
2302  CREATE_MATRIX_VECTOR_CONSECUTIVE(40) CREATE_MATRIX_VECTOR_CONSECUTIVE(42)
2303  CREATE_MATRIX_VECTOR_CONSECUTIVE(44) CREATE_MATRIX_VECTOR_CONSECUTIVE(46)
2304  CREATE_MATRIX_VECTOR_CONSECUTIVE(48) CREATE_MATRIX_VECTOR_CONSECUTIVE(50)
2305  CREATE_MATRIX_VECTOR_CONSECUTIVE(52) CREATE_MATRIX_VECTOR_CONSECUTIVE(54)
2306  CREATE_MATRIX_VECTOR_CONSECUTIVE(56) CREATE_MATRIX_VECTOR_CONSECUTIVE(58)
2307  CREATE_MATRIX_VECTOR_CONSECUTIVE(60) CREATE_MATRIX_VECTOR_CONSECUTIVE(62)
2308 
2309  // TBB parallel_for for consecutive case
2310  tbb::parallel_for(
2311  tbb::blocked_range<int>(0, matrix_size >> 3),
2312  [&](const tbb::blocked_range<int>& range) {
2313  for (int iter_idx = range.begin(); iter_idx != range.end(); ++iter_idx) {
2314  int base = 0;
2315  for (size_t i = 0; i < non_targets.size(); ++i) {
2316  if (iter_idx & (1ULL << i)) {
2317  base |= (1 << non_targets[i]);
2318  }
2319  }
2320 
2321  double* ptr_000 = (double*)input.get_data() + 2 * base;
2322  double* ptr_010 = (double*)input.get_data() + 2 * (base | index_step_middle);
2323  double* ptr_100 = (double*)input.get_data() + 2 * (base | index_step_outer);
2324  double* ptr_110 = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
2325 
2326  __m256d element_000_vec = _mm256_loadu_pd(ptr_000); // Loads indices[0] and [1]
2327  __m256d element_010_vec = _mm256_loadu_pd(ptr_010); // Loads indices[2] and [3]
2328  __m256d element_100_vec = _mm256_loadu_pd(ptr_100); // Loads indices[4] and [5]
2329  __m256d element_110_vec = _mm256_loadu_pd(ptr_110); // Loads indices[6] and [7]
2330 
2331  QGD_Complex16 results[8];
2332 
2333  int row_idx = 0;
2334  COMPUTE_3QBIT_ROW_CONSECUTIVE(u, 0, 2, 4, 6)
2335  row_idx = 1;
2336  COMPUTE_3QBIT_ROW_CONSECUTIVE(d, 8, 10, 12, 14)
2337  row_idx = 2;
2338  COMPUTE_3QBIT_ROW_CONSECUTIVE(e, 16, 18, 20, 22)
2339  row_idx = 3;
2340  COMPUTE_3QBIT_ROW_CONSECUTIVE(f, 24, 26, 28, 30)
2341  row_idx = 4;
2342  COMPUTE_3QBIT_ROW_CONSECUTIVE(g, 32, 34, 36, 38)
2343  row_idx = 5;
2344  COMPUTE_3QBIT_ROW_CONSECUTIVE(h, 40, 42, 44, 46)
2345  row_idx = 6;
2346  COMPUTE_3QBIT_ROW_CONSECUTIVE(i, 48, 50, 52, 54)
2347  row_idx = 7;
2348  COMPUTE_3QBIT_ROW_CONSECUTIVE(j, 56, 58, 60, 62)
2349 
2350  // Store results at the correct indices
2351  input[base] = results[0];
2352  input[base+1] = results[1];
2353  input[(base | index_step_middle)] = results[2];
2354  input[(base | index_step_middle)+1] = results[3];
2355  input[(base | index_step_outer)] = results[4];
2356  input[(base | index_step_outer)+1] = results[5];
2357  input[(base | index_step_middle | index_step_outer)] = results[6];
2358  input[(base | index_step_middle | index_step_outer)+1] = results[7];
2359  }
2360  }
2361  );
2362  }
2363  else {
2364  // Pre-compute all matrix vectors outside the parallel region
2365  CREATE_MATRIX_VECTOR(0) CREATE_MATRIX_VECTOR(1) CREATE_MATRIX_VECTOR(2) CREATE_MATRIX_VECTOR(3)
2366  CREATE_MATRIX_VECTOR(4) CREATE_MATRIX_VECTOR(5) CREATE_MATRIX_VECTOR(6) CREATE_MATRIX_VECTOR(7)
2367  CREATE_MATRIX_VECTOR(8) CREATE_MATRIX_VECTOR(9) CREATE_MATRIX_VECTOR(10) CREATE_MATRIX_VECTOR(11)
2368  CREATE_MATRIX_VECTOR(12) CREATE_MATRIX_VECTOR(13) CREATE_MATRIX_VECTOR(14) CREATE_MATRIX_VECTOR(15)
2369  CREATE_MATRIX_VECTOR(16) CREATE_MATRIX_VECTOR(17) CREATE_MATRIX_VECTOR(18) CREATE_MATRIX_VECTOR(19)
2370  CREATE_MATRIX_VECTOR(20) CREATE_MATRIX_VECTOR(21) CREATE_MATRIX_VECTOR(22) CREATE_MATRIX_VECTOR(23)
2371  CREATE_MATRIX_VECTOR(24) CREATE_MATRIX_VECTOR(25) CREATE_MATRIX_VECTOR(26) CREATE_MATRIX_VECTOR(27)
2372  CREATE_MATRIX_VECTOR(28) CREATE_MATRIX_VECTOR(29) CREATE_MATRIX_VECTOR(30) CREATE_MATRIX_VECTOR(31)
2373  CREATE_MATRIX_VECTOR(32) CREATE_MATRIX_VECTOR(33) CREATE_MATRIX_VECTOR(34) CREATE_MATRIX_VECTOR(35)
2374  CREATE_MATRIX_VECTOR(36) CREATE_MATRIX_VECTOR(37) CREATE_MATRIX_VECTOR(38) CREATE_MATRIX_VECTOR(39)
2375  CREATE_MATRIX_VECTOR(40) CREATE_MATRIX_VECTOR(41) CREATE_MATRIX_VECTOR(42) CREATE_MATRIX_VECTOR(43)
2376  CREATE_MATRIX_VECTOR(44) CREATE_MATRIX_VECTOR(45) CREATE_MATRIX_VECTOR(46) CREATE_MATRIX_VECTOR(47)
2377  CREATE_MATRIX_VECTOR(48) CREATE_MATRIX_VECTOR(49) CREATE_MATRIX_VECTOR(50) CREATE_MATRIX_VECTOR(51)
2378  CREATE_MATRIX_VECTOR(52) CREATE_MATRIX_VECTOR(53) CREATE_MATRIX_VECTOR(54) CREATE_MATRIX_VECTOR(55)
2379  CREATE_MATRIX_VECTOR(56) CREATE_MATRIX_VECTOR(57) CREATE_MATRIX_VECTOR(58) CREATE_MATRIX_VECTOR(59)
2380  CREATE_MATRIX_VECTOR(60) CREATE_MATRIX_VECTOR(61) CREATE_MATRIX_VECTOR(62) CREATE_MATRIX_VECTOR(63)
2381 
2382  // TBB parallel_for for non-consecutive case - note the step size of 2
2383  tbb::parallel_for(
2384  tbb::blocked_range<int>(0, matrix_size >> 3, 2), // grain_size of 2 to match iter_idx+=2
2385  [&](const tbb::blocked_range<int>& range) {
2386  for (int iter_idx = range.begin(); iter_idx < range.end(); iter_idx += 2) {
2387  int base = 0;
2388  for (size_t i = 0; i < non_targets.size(); ++i) {
2389  if (iter_idx & (1ULL << i)) {
2390  base |= (1 << non_targets[i]);
2391  }
2392  }
2393 
2394  double* element_outer = (double*)input.get_data() + 2 * base;
2395  double* element_inner = (double*)input.get_data() + 2 * (base | index_step_inner);
2396  double* element_middle = (double*)input.get_data() + 2 * (base | index_step_middle);
2397  double* element_middle_inner = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle);
2398  double* element_outer_pair = (double*)input.get_data() + 2 * (base | index_step_outer);
2399  double* element_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_outer);
2400  double* element_middle_pair = (double*)input.get_data() + 2 * (base | index_step_middle | index_step_outer);
2401  double* element_middle_inner_pair = (double*)input.get_data() + 2 * (base | index_step_inner | index_step_middle | index_step_outer);
2402 
2403  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
2404  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
2405  __m256d element_middle_vec = _mm256_loadu_pd(element_middle);
2406  __m256d element_middle_inner_vec = _mm256_loadu_pd(element_middle_inner);
2407  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
2408  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
2409  __m256d element_middle_pair_vec = _mm256_loadu_pd(element_middle_pair);
2410  __m256d element_middle_inner_pair_vec = _mm256_loadu_pd(element_middle_inner_pair);
2411 
2412  // Compute all 8 rows using macros
2413  COMPUTE_3QBIT_ROW(u, 0, 1, 2, 3, 4, 5, 6, 7)
2414  COMPUTE_3QBIT_ROW(d, 8, 9, 10, 11, 12, 13, 14, 15)
2415  COMPUTE_3QBIT_ROW(e, 16, 17, 18, 19, 20, 21, 22, 23)
2416  COMPUTE_3QBIT_ROW(f, 24, 25, 26, 27, 28, 29, 30, 31)
2417  COMPUTE_3QBIT_ROW(g, 32, 33, 34, 35, 36, 37, 38, 39)
2418  COMPUTE_3QBIT_ROW(h, 40, 41, 42, 43, 44, 45, 46, 47)
2419  COMPUTE_3QBIT_ROW(i, 48, 49, 50, 51, 52, 53, 54, 55)
2420  COMPUTE_3QBIT_ROW(j, 56, 57, 58, 59, 60, 61, 62, 63)
2421 
2422  // Store results
2423  _mm256_storeu_pd(element_outer, data_u);
2424  _mm256_storeu_pd(element_inner, data_d);
2425  _mm256_storeu_pd(element_middle, data_e);
2426  _mm256_storeu_pd(element_middle_inner, data_f);
2427  _mm256_storeu_pd(element_outer_pair, data_g);
2428  _mm256_storeu_pd(element_inner_pair, data_h);
2429  _mm256_storeu_pd(element_middle_pair, data_i);
2430  _mm256_storeu_pd(element_middle_inner_pair, data_j);
2431  }
2432  }
2433  );
2434  }
2435 }
2443 void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
2444 
2445  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
2446 
2447  int index_step_inner = 1 << involved_qbits[0];
2448  int index_step_middle1 = 1 << involved_qbits[1];
2449  int index_step_middle2 = 1 << involved_qbits[2];
2450  int index_step_outer = 1 << involved_qbits[3];
2451 
2452  // Properly iterate through all blocks
2453  int num_qubits = (int)std::log2(matrix_size);
2454  int num_blocks = matrix_size >> 4; // 2^4 = 16 elements per block
2455 
2456  // Identify non-involved qubits
2457  std::vector<int> is_target(num_qubits, 0);
2458  for (int q : involved_qbits) is_target[q] = 1;
2459 
2460  std::vector<int> non_targets;
2461  non_targets.reserve(num_qubits - 4);
2462  for (int q = 0; q < num_qubits; ++q) {
2463  if (!is_target[q]) non_targets.push_back(q);
2464  }
2465 
2466  #pragma omp parallel for schedule(static)
2467  for (int block_idx = 0; block_idx < num_blocks; block_idx++) {
2468  __m256d element_0000_vec_real = _mm256_setzero_pd();
2469  __m256d element_0010_vec_real = _mm256_setzero_pd();
2470  __m256d element_0100_vec_real = _mm256_setzero_pd();
2471  __m256d element_0110_vec_real = _mm256_setzero_pd();
2472  __m256d element_1000_vec_real = _mm256_setzero_pd();
2473  __m256d element_1010_vec_real = _mm256_setzero_pd();
2474  __m256d element_1100_vec_real = _mm256_setzero_pd();
2475  __m256d element_1110_vec_real = _mm256_setzero_pd();
2476 
2477  int base = 0;
2478  for (size_t i = 0; i < non_targets.size(); ++i) {
2479  if (block_idx & (1ULL << i)) {
2480  base |= (1 << non_targets[i]);
2481  }
2482  }
2483 
2484  int current_idx_outer_loc = base;
2485  int current_idx_inner_loc = base | index_step_inner;
2486  int current_idx_middle1_loc = base | index_step_middle1;
2487  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
2488  int current_idx_middle2_loc = base | index_step_middle2;
2489  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
2490  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
2491  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
2492 
2493  int current_idx_outer_pair_loc = base | index_step_outer;
2494  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
2495  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
2496  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
2497  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
2498  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
2499  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
2500  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
2501 
2502  if(involved_qbits[0] == 0){
2503  //preload all 16 elements instead of the unitary kernel matrix
2504  element_0000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
2505  element_0010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
2506  element_0100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
2507  element_0110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
2508  element_1000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
2509  element_1010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
2510  element_1100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
2511  element_1110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
2512  }
2513  else{
2514  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
2515  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
2516 
2517  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
2518  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
2519 
2520  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
2521  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
2522 
2523  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
2524  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
2525 
2526  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
2527  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
2528 
2529  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
2530  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
2531 
2532  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
2533  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
2534 
2535  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
2536  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
2537 
2538  element_0000_vec_real = get_AVX_vector(element_outer, element_inner);
2539  element_0010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
2540  element_0100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
2541  element_0110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
2542  element_1000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
2543  element_1010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
2544  element_1100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
2545  element_1110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
2546  }
2547 
2548  __m256d element_0000_vec_imag = _mm256_permute4x64_pd(element_0000_vec_real, 0b10110001);
2549  __m256d element_0010_vec_imag = _mm256_permute4x64_pd(element_0010_vec_real, 0b10110001);
2550  __m256d element_0100_vec_imag = _mm256_permute4x64_pd(element_0100_vec_real, 0b10110001);
2551  __m256d element_0110_vec_imag = _mm256_permute4x64_pd(element_0110_vec_real, 0b10110001);
2552  __m256d element_1000_vec_imag = _mm256_permute4x64_pd(element_1000_vec_real, 0b10110001);
2553  __m256d element_1010_vec_imag = _mm256_permute4x64_pd(element_1010_vec_real, 0b10110001);
2554  __m256d element_1100_vec_imag = _mm256_permute4x64_pd(element_1100_vec_real, 0b10110001);
2555  __m256d element_1110_vec_imag = _mm256_permute4x64_pd(element_1110_vec_real, 0b10110001);
2556 
2557  element_0000_vec_real = _mm256_mul_pd(element_0000_vec_real,neg);
2558  element_0010_vec_real = _mm256_mul_pd(element_0010_vec_real,neg);
2559  element_0100_vec_real = _mm256_mul_pd(element_0100_vec_real,neg);
2560  element_0110_vec_real = _mm256_mul_pd(element_0110_vec_real,neg);
2561  element_1000_vec_real = _mm256_mul_pd(element_1000_vec_real,neg);
2562  element_1010_vec_real = _mm256_mul_pd(element_1010_vec_real,neg);
2563  element_1100_vec_real = _mm256_mul_pd(element_1100_vec_real,neg);
2564  element_1110_vec_real = _mm256_mul_pd(element_1110_vec_real,neg);
2565 
2566  QGD_Complex16 results[16];
2567  for (int mult_idx = 0; mult_idx < 16; mult_idx++) {
2568  double* unitary_row_1 = (double*)unitary.get_data() + 32*mult_idx;
2569  double* unitary_row_2 = unitary_row_1 + 4;
2570  double* unitary_row_3 = unitary_row_1 + 8;
2571  double* unitary_row_4 = unitary_row_1 + 12;
2572  double* unitary_row_5 = unitary_row_1 + 16;
2573  double* unitary_row_6 = unitary_row_1 + 20;
2574  double* unitary_row_7 = unitary_row_1 + 24;
2575  double* unitary_row_8 = unitary_row_1 + 28;
2576 
2577  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
2578  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
2579  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
2580  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
2581  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
2582  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
2583  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
2584  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
2585 
2586  __m256d data_real = _mm256_setzero_pd();
2587  __m256d data_imag = _mm256_setzero_pd();
2588 
2589  data_real = _mm256_fmadd_pd(element_0000_vec_real, row1_vec, data_real);
2590  data_imag = _mm256_fmadd_pd(element_0000_vec_imag, row1_vec, data_imag);
2591  data_real = _mm256_fmadd_pd(element_0010_vec_real, row2_vec, data_real);
2592  data_imag = _mm256_fmadd_pd(element_0010_vec_imag, row2_vec, data_imag);
2593  data_real = _mm256_fmadd_pd(element_0100_vec_real, row3_vec, data_real);
2594  data_imag = _mm256_fmadd_pd(element_0100_vec_imag, row3_vec, data_imag);
2595  data_real = _mm256_fmadd_pd(element_0110_vec_real, row4_vec, data_real);
2596  data_imag = _mm256_fmadd_pd(element_0110_vec_imag, row4_vec, data_imag);
2597  data_real = _mm256_fmadd_pd(element_1000_vec_real, row5_vec, data_real);
2598  data_imag = _mm256_fmadd_pd(element_1000_vec_imag, row5_vec, data_imag);
2599  data_real = _mm256_fmadd_pd(element_1010_vec_real, row6_vec, data_real);
2600  data_imag = _mm256_fmadd_pd(element_1010_vec_imag, row6_vec, data_imag);
2601  data_real = _mm256_fmadd_pd(element_1100_vec_real, row7_vec, data_real);
2602  data_imag = _mm256_fmadd_pd(element_1100_vec_imag, row7_vec, data_imag);
2603  data_real = _mm256_fmadd_pd(element_1110_vec_real, row8_vec, data_real);
2604  data_imag = _mm256_fmadd_pd(element_1110_vec_imag, row8_vec, data_imag);
2605 
2606  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
2607  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
2608  final_vec = _mm256_hadd_pd(final_vec, final_vec);
2609  __m128d low128 = _mm256_castpd256_pd128(final_vec);
2610  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
2611  results[mult_idx].real = _mm_cvtsd_f64(low128);
2612  results[mult_idx].imag = _mm_cvtsd_f64(high128);
2613  }
2614 
2615  input[current_idx_outer_loc] = results[0];
2616  input[current_idx_inner_loc] = results[1];
2617  input[current_idx_middle1_loc] = results[2];
2618  input[current_idx_middle1_inner_loc] = results[3];
2619  input[current_idx_middle2_loc] = results[4];
2620  input[current_idx_middle2_inner_loc] = results[5];
2621  input[current_idx_middle12_loc] = results[6];
2622  input[current_idx_middle12_inner_loc] = results[7];
2623  input[current_idx_outer_pair_loc] = results[8];
2624  input[current_idx_inner_pair_loc] = results[9];
2625  input[current_idx_middle1_pair_loc] = results[10];
2626  input[current_idx_middle1_inner_pair_loc] = results[11];
2627  input[current_idx_middle2_pair_loc] = results[12];
2628  input[current_idx_middle2_inner_pair_loc] = results[13];
2629  input[current_idx_middle12_pair_loc] = results[14];
2630  input[current_idx_middle12_inner_pair_loc] = results[15];
2631  }
2632 }
2640 void apply_4qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
2641 
2642  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
2643 
2644  int index_step_inner = 1 << involved_qbits[0];
2645  int index_step_middle1 = 1 << involved_qbits[1];
2646  int index_step_middle2 = 1 << involved_qbits[2];
2647  int index_step_outer = 1 << involved_qbits[3];
2648 
2649  // Properly iterate through all blocks
2650  int num_qubits = (int)std::log2(matrix_size);
2651 
2652  // Identify non-involved qubits
2653  std::vector<int> is_target(num_qubits, 0);
2654  for (int q : involved_qbits) is_target[q] = 1;
2655 
2656  std::vector<int> non_targets;
2657  non_targets.reserve(num_qubits - 4);
2658  for (int q = 0; q < num_qubits; ++q) {
2659  if (!is_target[q]) non_targets.push_back(q);
2660  }
2661 
2662  tbb::parallel_for(
2663  tbb::blocked_range<int>(0, matrix_size >> 4),
2664  [&](const tbb::blocked_range<int>& range) {
2665  for (int block_idx = range.begin(); block_idx != range.end(); ++block_idx) {
2666  __m256d element_0000_vec_real = _mm256_setzero_pd();
2667  __m256d element_0010_vec_real = _mm256_setzero_pd();
2668  __m256d element_0100_vec_real = _mm256_setzero_pd();
2669  __m256d element_0110_vec_real = _mm256_setzero_pd();
2670  __m256d element_1000_vec_real = _mm256_setzero_pd();
2671  __m256d element_1010_vec_real = _mm256_setzero_pd();
2672  __m256d element_1100_vec_real = _mm256_setzero_pd();
2673  __m256d element_1110_vec_real = _mm256_setzero_pd();
2674 
2675  int base = 0;
2676  for (size_t i = 0; i < non_targets.size(); ++i) {
2677  if (block_idx & (1ULL << i)) {
2678  base |= (1 << non_targets[i]);
2679  }
2680  }
2681 
2682  int current_idx_outer_loc = base;
2683  int current_idx_inner_loc = base | index_step_inner;
2684  int current_idx_middle1_loc = base | index_step_middle1;
2685  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
2686  int current_idx_middle2_loc = base | index_step_middle2;
2687  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
2688  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
2689  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
2690 
2691  int current_idx_outer_pair_loc = base | index_step_outer;
2692  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
2693  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
2694  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
2695  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
2696  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
2697  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
2698  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
2699 
2700  if(involved_qbits[0] == 0){
2701  //preload all 16 elements instead of the unitary kernel matrix
2702  element_0000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
2703  element_0010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
2704  element_0100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
2705  element_0110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
2706  element_1000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
2707  element_1010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
2708  element_1100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
2709  element_1110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
2710  }
2711  else{
2712  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
2713  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
2714 
2715  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
2716  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
2717 
2718  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
2719  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
2720 
2721  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
2722  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
2723 
2724  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
2725  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
2726 
2727  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
2728  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
2729 
2730  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
2731  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
2732 
2733  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
2734  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
2735 
2736  element_0000_vec_real = get_AVX_vector(element_outer, element_inner);
2737  element_0010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
2738  element_0100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
2739  element_0110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
2740  element_1000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
2741  element_1010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
2742  element_1100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
2743  element_1110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
2744  }
2745 
2746  __m256d element_0000_vec_imag = _mm256_permute4x64_pd(element_0000_vec_real, 0b10110001);
2747  __m256d element_0010_vec_imag = _mm256_permute4x64_pd(element_0010_vec_real, 0b10110001);
2748  __m256d element_0100_vec_imag = _mm256_permute4x64_pd(element_0100_vec_real, 0b10110001);
2749  __m256d element_0110_vec_imag = _mm256_permute4x64_pd(element_0110_vec_real, 0b10110001);
2750  __m256d element_1000_vec_imag = _mm256_permute4x64_pd(element_1000_vec_real, 0b10110001);
2751  __m256d element_1010_vec_imag = _mm256_permute4x64_pd(element_1010_vec_real, 0b10110001);
2752  __m256d element_1100_vec_imag = _mm256_permute4x64_pd(element_1100_vec_real, 0b10110001);
2753  __m256d element_1110_vec_imag = _mm256_permute4x64_pd(element_1110_vec_real, 0b10110001);
2754 
2755  element_0000_vec_real = _mm256_mul_pd(element_0000_vec_real,neg);
2756  element_0010_vec_real = _mm256_mul_pd(element_0010_vec_real,neg);
2757  element_0100_vec_real = _mm256_mul_pd(element_0100_vec_real,neg);
2758  element_0110_vec_real = _mm256_mul_pd(element_0110_vec_real,neg);
2759  element_1000_vec_real = _mm256_mul_pd(element_1000_vec_real,neg);
2760  element_1010_vec_real = _mm256_mul_pd(element_1010_vec_real,neg);
2761  element_1100_vec_real = _mm256_mul_pd(element_1100_vec_real,neg);
2762  element_1110_vec_real = _mm256_mul_pd(element_1110_vec_real,neg);
2763 
2764  QGD_Complex16 results[16];
2765  for (int mult_idx = 0; mult_idx < 16; mult_idx++) {
2766  double* unitary_row_1 = (double*)unitary.get_data() + 32*mult_idx;
2767  double* unitary_row_2 = unitary_row_1 + 4;
2768  double* unitary_row_3 = unitary_row_1 + 8;
2769  double* unitary_row_4 = unitary_row_1 + 12;
2770  double* unitary_row_5 = unitary_row_1 + 16;
2771  double* unitary_row_6 = unitary_row_1 + 20;
2772  double* unitary_row_7 = unitary_row_1 + 24;
2773  double* unitary_row_8 = unitary_row_1 + 28;
2774 
2775  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
2776  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
2777  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
2778  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
2779  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
2780  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
2781  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
2782  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
2783 
2784  __m256d data_real = _mm256_setzero_pd();
2785  __m256d data_imag = _mm256_setzero_pd();
2786 
2787  data_real = _mm256_fmadd_pd(element_0000_vec_real, row1_vec, data_real);
2788  data_imag = _mm256_fmadd_pd(element_0000_vec_imag, row1_vec, data_imag);
2789  data_real = _mm256_fmadd_pd(element_0010_vec_real, row2_vec, data_real);
2790  data_imag = _mm256_fmadd_pd(element_0010_vec_imag, row2_vec, data_imag);
2791  data_real = _mm256_fmadd_pd(element_0100_vec_real, row3_vec, data_real);
2792  data_imag = _mm256_fmadd_pd(element_0100_vec_imag, row3_vec, data_imag);
2793  data_real = _mm256_fmadd_pd(element_0110_vec_real, row4_vec, data_real);
2794  data_imag = _mm256_fmadd_pd(element_0110_vec_imag, row4_vec, data_imag);
2795  data_real = _mm256_fmadd_pd(element_1000_vec_real, row5_vec, data_real);
2796  data_imag = _mm256_fmadd_pd(element_1000_vec_imag, row5_vec, data_imag);
2797  data_real = _mm256_fmadd_pd(element_1010_vec_real, row6_vec, data_real);
2798  data_imag = _mm256_fmadd_pd(element_1010_vec_imag, row6_vec, data_imag);
2799  data_real = _mm256_fmadd_pd(element_1100_vec_real, row7_vec, data_real);
2800  data_imag = _mm256_fmadd_pd(element_1100_vec_imag, row7_vec, data_imag);
2801  data_real = _mm256_fmadd_pd(element_1110_vec_real, row8_vec, data_real);
2802  data_imag = _mm256_fmadd_pd(element_1110_vec_imag, row8_vec, data_imag);
2803 
2804  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
2805  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
2806  final_vec = _mm256_hadd_pd(final_vec, final_vec);
2807  __m128d low128 = _mm256_castpd256_pd128(final_vec);
2808  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
2809  results[mult_idx].real = _mm_cvtsd_f64(low128);
2810  results[mult_idx].imag = _mm_cvtsd_f64(high128);
2811  }
2812 
2813  input[current_idx_outer_loc] = results[0];
2814  input[current_idx_inner_loc] = results[1];
2815  input[current_idx_middle1_loc] = results[2];
2816  input[current_idx_middle1_inner_loc] = results[3];
2817  input[current_idx_middle2_loc] = results[4];
2818  input[current_idx_middle2_inner_loc] = results[5];
2819  input[current_idx_middle12_loc] = results[6];
2820  input[current_idx_middle12_inner_loc] = results[7];
2821  input[current_idx_outer_pair_loc] = results[8];
2822  input[current_idx_inner_pair_loc] = results[9];
2823  input[current_idx_middle1_pair_loc] = results[10];
2824  input[current_idx_middle1_inner_pair_loc] = results[11];
2825  input[current_idx_middle2_pair_loc] = results[12];
2826  input[current_idx_middle2_inner_pair_loc] = results[13];
2827  input[current_idx_middle12_pair_loc] = results[14];
2828  input[current_idx_middle12_inner_pair_loc] = results[15];
2829  }
2830  }
2831  );
2832 
2833 }
2834 
2842 void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
2843  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
2844 
2845  int index_step_inner = 1 << involved_qbits[0];
2846  int index_step_middle1 = 1 << involved_qbits[1];
2847  int index_step_middle2 = 1 << involved_qbits[2];
2848  int index_step_middle3 = 1 << involved_qbits[3];
2849  int index_step_outer = 1 << involved_qbits[4];
2850 
2851  int num_qubits = (int)std::log2(matrix_size);
2852  int num_blocks = matrix_size >> 5;
2853  std::vector<int> is_target(num_qubits, 0);
2854  for (int q : involved_qbits) is_target[q] = 1;
2855 
2856  std::vector<int> non_targets;
2857  non_targets.reserve(num_qubits - 5);
2858  for (int q = 0; q < num_qubits; ++q) {
2859  if (!is_target[q]) non_targets.push_back(q);
2860  }
2861  #pragma omp parallel for schedule(static)
2862  for (int block_idx = 0; block_idx < num_blocks; block_idx++) {
2863  // Calculate base index properly
2864  int base = 0;
2865  for (size_t i = 0; i < non_targets.size(); ++i) {
2866  if (block_idx & (1ULL << i)) {
2867  base |= (1 << non_targets[i]);
2868  }
2869  }
2870  // Calculate all 32 indices using OR operations
2871  int current_idx_outer_loc = base;
2872  int current_idx_inner_loc = base | index_step_inner;
2873 
2874  int current_idx_middle1_loc = base | index_step_middle1;
2875  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
2876 
2877  int current_idx_middle2_loc = base | index_step_middle2;
2878  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
2879 
2880  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
2881  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
2882 
2883  int current_idx_middle3_loc = base | index_step_middle3;
2884  int current_idx_middle3_inner_loc = base | index_step_middle3 | index_step_inner;
2885 
2886  int current_idx_middle13_loc = base | index_step_middle1 | index_step_middle3;
2887  int current_idx_middle13_inner_loc = base | index_step_middle1 | index_step_middle3 | index_step_inner;
2888 
2889  int current_idx_middle23_loc = base | index_step_middle2 | index_step_middle3;
2890  int current_idx_middle23_inner_loc = base | index_step_middle2 | index_step_middle3 | index_step_inner;
2891 
2892  int current_idx_middle123_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3;
2893  int current_idx_middle123_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
2894 
2895  int current_idx_outer_pair_loc = base | index_step_outer;
2896  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
2897 
2898  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
2899  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
2900 
2901  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
2902  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
2903 
2904  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
2905  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
2906 
2907  int current_idx_middle3_pair_loc = base | index_step_outer | index_step_middle3;
2908  int current_idx_middle3_inner_pair_loc = base | index_step_outer | index_step_middle3 | index_step_inner;
2909 
2910  int current_idx_middle13_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3;
2911  int current_idx_middle13_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3 | index_step_inner;
2912 
2913  int current_idx_middle23_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3;
2914  int current_idx_middle23_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3 | index_step_inner;
2915 
2916  int current_idx_middle123_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3;
2917  int current_idx_middle123_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
2918 
2919  __m256d element_00000_vec_real = _mm256_setzero_pd();
2920  __m256d element_00010_vec_real = _mm256_setzero_pd();
2921  __m256d element_00100_vec_real = _mm256_setzero_pd();
2922  __m256d element_00110_vec_real = _mm256_setzero_pd();
2923  __m256d element_01000_vec_real = _mm256_setzero_pd();
2924  __m256d element_01010_vec_real = _mm256_setzero_pd();
2925  __m256d element_01100_vec_real = _mm256_setzero_pd();
2926  __m256d element_01110_vec_real = _mm256_setzero_pd();
2927  __m256d element_10000_vec_real = _mm256_setzero_pd();
2928  __m256d element_10010_vec_real = _mm256_setzero_pd();
2929  __m256d element_10100_vec_real = _mm256_setzero_pd();
2930  __m256d element_10110_vec_real = _mm256_setzero_pd();
2931  __m256d element_11000_vec_real = _mm256_setzero_pd();
2932  __m256d element_11010_vec_real = _mm256_setzero_pd();
2933  __m256d element_11100_vec_real = _mm256_setzero_pd();
2934  __m256d element_11110_vec_real = _mm256_setzero_pd();
2935 
2936  if (involved_qbits[0] == 0) {
2937  // Preload all 32 elements instead of the unitary kernel matrix
2938  element_00000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
2939  element_00010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
2940  element_00100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
2941  element_00110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
2942  element_01000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_loc);
2943  element_01010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_loc);
2944  element_01100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_loc);
2945  element_01110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_loc);
2946  element_10000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
2947  element_10010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
2948  element_10100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
2949  element_10110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
2950  element_11000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_pair_loc);
2951  element_11010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_pair_loc);
2952  element_11100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_pair_loc);
2953  element_11110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_pair_loc);
2954 
2955  }
2956  else {
2957  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
2958  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
2959 
2960  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
2961  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
2962 
2963  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
2964  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
2965 
2966  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
2967  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
2968 
2969  double* element_middle3 = (double*)input.get_data() + 2 * current_idx_middle3_loc;
2970  double* element_middle3_inner = (double*)input.get_data() + 2 * current_idx_middle3_inner_loc;
2971 
2972  double* element_middle13 = (double*)input.get_data() + 2 * current_idx_middle13_loc;
2973  double* element_middle13_inner = (double*)input.get_data() + 2 * current_idx_middle13_inner_loc;
2974 
2975  double* element_middle23 = (double*)input.get_data() + 2 * current_idx_middle23_loc;
2976  double* element_middle23_inner = (double*)input.get_data() + 2 * current_idx_middle23_inner_loc;
2977 
2978  double* element_middle123 = (double*)input.get_data() + 2 * current_idx_middle123_loc;
2979  double* element_middle123_inner = (double*)input.get_data() + 2 * current_idx_middle123_inner_loc;
2980 
2981  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
2982  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
2983 
2984  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
2985  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
2986 
2987  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
2988  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
2989 
2990  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
2991  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
2992 
2993  double* element_middle3_pair = (double*)input.get_data() + 2 * current_idx_middle3_pair_loc;
2994  double* element_middle3_inner_pair = (double*)input.get_data() + 2 * current_idx_middle3_inner_pair_loc;
2995 
2996  double* element_middle13_pair = (double*)input.get_data() + 2 * current_idx_middle13_pair_loc;
2997  double* element_middle13_inner_pair = (double*)input.get_data() + 2 * current_idx_middle13_inner_pair_loc;
2998 
2999  double* element_middle23_pair = (double*)input.get_data() + 2 * current_idx_middle23_pair_loc;
3000  double* element_middle23_inner_pair = (double*)input.get_data() + 2 * current_idx_middle23_inner_pair_loc;
3001 
3002  double* element_middle123_pair = (double*)input.get_data() + 2 * current_idx_middle123_pair_loc;
3003  double* element_middle123_inner_pair = (double*)input.get_data() + 2 * current_idx_middle123_inner_pair_loc;
3004 
3005  element_00000_vec_real = get_AVX_vector(element_outer, element_inner);
3006  element_00010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
3007  element_00100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
3008  element_00110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
3009  element_01000_vec_real = get_AVX_vector(element_middle3, element_middle3_inner);
3010  element_01010_vec_real = get_AVX_vector(element_middle13, element_middle13_inner);
3011  element_01100_vec_real = get_AVX_vector(element_middle23, element_middle23_inner);
3012  element_01110_vec_real = get_AVX_vector(element_middle123, element_middle123_inner);
3013  element_10000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
3014  element_10010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
3015  element_10100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
3016  element_10110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
3017  element_11000_vec_real = get_AVX_vector(element_middle3_pair, element_middle3_inner_pair);
3018  element_11010_vec_real = get_AVX_vector(element_middle13_pair, element_middle13_inner_pair);
3019  element_11100_vec_real = get_AVX_vector(element_middle23_pair, element_middle23_inner_pair);
3020  element_11110_vec_real = get_AVX_vector(element_middle123_pair, element_middle123_inner_pair);
3021 
3022  }
3023  __m256d element_00000_vec_imag = _mm256_permute4x64_pd(element_00000_vec_real, 0b10110001);
3024  __m256d element_00010_vec_imag = _mm256_permute4x64_pd(element_00010_vec_real, 0b10110001);
3025  __m256d element_00100_vec_imag = _mm256_permute4x64_pd(element_00100_vec_real, 0b10110001);
3026  __m256d element_00110_vec_imag = _mm256_permute4x64_pd(element_00110_vec_real, 0b10110001);
3027  __m256d element_01000_vec_imag = _mm256_permute4x64_pd(element_01000_vec_real, 0b10110001);
3028  __m256d element_01010_vec_imag = _mm256_permute4x64_pd(element_01010_vec_real, 0b10110001);
3029  __m256d element_01100_vec_imag = _mm256_permute4x64_pd(element_01100_vec_real, 0b10110001);
3030  __m256d element_01110_vec_imag = _mm256_permute4x64_pd(element_01110_vec_real, 0b10110001);
3031  __m256d element_10000_vec_imag = _mm256_permute4x64_pd(element_10000_vec_real, 0b10110001);
3032  __m256d element_10010_vec_imag = _mm256_permute4x64_pd(element_10010_vec_real, 0b10110001);
3033  __m256d element_10100_vec_imag = _mm256_permute4x64_pd(element_10100_vec_real, 0b10110001);
3034  __m256d element_10110_vec_imag = _mm256_permute4x64_pd(element_10110_vec_real, 0b10110001);
3035  __m256d element_11000_vec_imag = _mm256_permute4x64_pd(element_11000_vec_real, 0b10110001);
3036  __m256d element_11010_vec_imag = _mm256_permute4x64_pd(element_11010_vec_real, 0b10110001);
3037  __m256d element_11100_vec_imag = _mm256_permute4x64_pd(element_11100_vec_real, 0b10110001);
3038  __m256d element_11110_vec_imag = _mm256_permute4x64_pd(element_11110_vec_real, 0b10110001);
3039 
3040  element_00000_vec_real = _mm256_mul_pd(element_00000_vec_real,neg);
3041  element_00010_vec_real = _mm256_mul_pd(element_00010_vec_real,neg);
3042  element_00100_vec_real = _mm256_mul_pd(element_00100_vec_real,neg);
3043  element_00110_vec_real = _mm256_mul_pd(element_00110_vec_real,neg);
3044  element_01000_vec_real = _mm256_mul_pd(element_01000_vec_real,neg);
3045  element_01010_vec_real = _mm256_mul_pd(element_01010_vec_real,neg);
3046  element_01100_vec_real = _mm256_mul_pd(element_01100_vec_real,neg);
3047  element_01110_vec_real = _mm256_mul_pd(element_01110_vec_real,neg);
3048  element_10000_vec_real = _mm256_mul_pd(element_10000_vec_real,neg);
3049  element_10010_vec_real = _mm256_mul_pd(element_10010_vec_real,neg);
3050  element_10100_vec_real = _mm256_mul_pd(element_10100_vec_real,neg);
3051  element_10110_vec_real = _mm256_mul_pd(element_10110_vec_real,neg);
3052  element_11000_vec_real = _mm256_mul_pd(element_11000_vec_real,neg);
3053  element_11010_vec_real = _mm256_mul_pd(element_11010_vec_real,neg);
3054  element_11100_vec_real = _mm256_mul_pd(element_11100_vec_real,neg);
3055  element_11110_vec_real = _mm256_mul_pd(element_11110_vec_real,neg);
3056 
3057  QGD_Complex16 results[32];
3058  for (int mult_idx = 0; mult_idx < 32; mult_idx++) {
3059  double* unitary_row_1 = (double*)unitary.get_data() + 64*mult_idx;
3060  double* unitary_row_2 = unitary_row_1 + 4;
3061  double* unitary_row_3 = unitary_row_1 + 8;
3062  double* unitary_row_4 = unitary_row_1 + 12;
3063  double* unitary_row_5 = unitary_row_1 + 16;
3064  double* unitary_row_6 = unitary_row_1 + 20;
3065  double* unitary_row_7 = unitary_row_1 + 24;
3066  double* unitary_row_8 = unitary_row_1 + 28;
3067  double* unitary_row_9 = unitary_row_1 + 32;
3068  double* unitary_row_10 = unitary_row_1 + 36;
3069  double* unitary_row_11 = unitary_row_1 + 40;
3070  double* unitary_row_12 = unitary_row_1 + 44;
3071  double* unitary_row_13 = unitary_row_1 + 48;
3072  double* unitary_row_14 = unitary_row_1 + 52;
3073  double* unitary_row_15 = unitary_row_1 + 56;
3074  double* unitary_row_16 = unitary_row_1 + 60;
3075 
3076  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
3077  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
3078  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
3079  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
3080  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
3081  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
3082  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
3083  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
3084  __m256d row9_vec = _mm256_loadu_pd(unitary_row_9);
3085  __m256d row10_vec = _mm256_loadu_pd(unitary_row_10);
3086  __m256d row11_vec = _mm256_loadu_pd(unitary_row_11);
3087  __m256d row12_vec = _mm256_loadu_pd(unitary_row_12);
3088  __m256d row13_vec = _mm256_loadu_pd(unitary_row_13);
3089  __m256d row14_vec = _mm256_loadu_pd(unitary_row_14);
3090  __m256d row15_vec = _mm256_loadu_pd(unitary_row_15);
3091  __m256d row16_vec = _mm256_loadu_pd(unitary_row_16);
3092 
3093  __m256d data_real = _mm256_setzero_pd();
3094  __m256d data_imag = _mm256_setzero_pd();
3095 
3096  data_real = _mm256_fmadd_pd(element_00000_vec_real, row1_vec, data_real);
3097  data_imag = _mm256_fmadd_pd(element_00000_vec_imag, row1_vec, data_imag);
3098  data_real = _mm256_fmadd_pd(element_00010_vec_real, row2_vec, data_real);
3099  data_imag = _mm256_fmadd_pd(element_00010_vec_imag, row2_vec, data_imag);
3100  data_real = _mm256_fmadd_pd(element_00100_vec_real, row3_vec, data_real);
3101  data_imag = _mm256_fmadd_pd(element_00100_vec_imag, row3_vec, data_imag);
3102  data_real = _mm256_fmadd_pd(element_00110_vec_real, row4_vec, data_real);
3103  data_imag = _mm256_fmadd_pd(element_00110_vec_imag, row4_vec, data_imag);
3104  data_real = _mm256_fmadd_pd(element_01000_vec_real, row5_vec, data_real);
3105  data_imag = _mm256_fmadd_pd(element_01000_vec_imag, row5_vec, data_imag);
3106  data_real = _mm256_fmadd_pd(element_01010_vec_real, row6_vec, data_real);
3107  data_imag = _mm256_fmadd_pd(element_01010_vec_imag, row6_vec, data_imag);
3108  data_real = _mm256_fmadd_pd(element_01100_vec_real, row7_vec, data_real);
3109  data_imag = _mm256_fmadd_pd(element_01100_vec_imag, row7_vec, data_imag);
3110  data_real = _mm256_fmadd_pd(element_01110_vec_real, row8_vec, data_real);
3111  data_imag = _mm256_fmadd_pd(element_01110_vec_imag, row8_vec, data_imag);
3112  data_real = _mm256_fmadd_pd(element_10000_vec_real, row9_vec, data_real);
3113  data_imag = _mm256_fmadd_pd(element_10000_vec_imag, row9_vec, data_imag);
3114  data_real = _mm256_fmadd_pd(element_10010_vec_real, row10_vec, data_real);
3115  data_imag = _mm256_fmadd_pd(element_10010_vec_imag, row10_vec, data_imag);
3116  data_real = _mm256_fmadd_pd(element_10100_vec_real, row11_vec, data_real);
3117  data_imag = _mm256_fmadd_pd(element_10100_vec_imag, row11_vec, data_imag);
3118  data_real = _mm256_fmadd_pd(element_10110_vec_real, row12_vec, data_real);
3119  data_imag = _mm256_fmadd_pd(element_10110_vec_imag, row12_vec, data_imag);
3120  data_real = _mm256_fmadd_pd(element_11000_vec_real, row13_vec, data_real);
3121  data_imag = _mm256_fmadd_pd(element_11000_vec_imag, row13_vec, data_imag);
3122  data_real = _mm256_fmadd_pd(element_11010_vec_real, row14_vec, data_real);
3123  data_imag = _mm256_fmadd_pd(element_11010_vec_imag, row14_vec, data_imag);
3124  data_real = _mm256_fmadd_pd(element_11100_vec_real, row15_vec, data_real);
3125  data_imag = _mm256_fmadd_pd(element_11100_vec_imag, row15_vec, data_imag);
3126  data_real = _mm256_fmadd_pd(element_11110_vec_real, row16_vec, data_real);
3127  data_imag = _mm256_fmadd_pd(element_11110_vec_imag, row16_vec, data_imag);
3128 
3129  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
3130  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
3131  final_vec = _mm256_hadd_pd(final_vec, final_vec);
3132  __m128d low128 = _mm256_castpd256_pd128(final_vec);
3133  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
3134  results[mult_idx].real = _mm_cvtsd_f64(low128);
3135  results[mult_idx].imag = _mm_cvtsd_f64(high128);
3136  }
3137  input[current_idx_outer_loc] = results[0];
3138  input[current_idx_inner_loc] = results[1];
3139  input[current_idx_middle1_loc] = results[2];
3140  input[current_idx_middle1_inner_loc] = results[3];
3141  input[current_idx_middle2_loc] = results[4];
3142  input[current_idx_middle2_inner_loc] = results[5];
3143  input[current_idx_middle12_loc] = results[6];
3144  input[current_idx_middle12_inner_loc] = results[7];
3145  input[current_idx_middle3_loc] = results[8];
3146  input[current_idx_middle3_inner_loc] = results[9];
3147  input[current_idx_middle13_loc] = results[10];
3148  input[current_idx_middle13_inner_loc] = results[11];
3149  input[current_idx_middle23_loc] = results[12];
3150  input[current_idx_middle23_inner_loc] = results[13];
3151  input[current_idx_middle123_loc] = results[14];
3152  input[current_idx_middle123_inner_loc] = results[15];
3153  input[current_idx_outer_pair_loc] = results[16];
3154  input[current_idx_inner_pair_loc] = results[17];
3155  input[current_idx_middle1_pair_loc] = results[18];
3156  input[current_idx_middle1_inner_pair_loc] = results[19];
3157  input[current_idx_middle2_pair_loc] = results[20];
3158  input[current_idx_middle2_inner_pair_loc] = results[21];
3159  input[current_idx_middle12_pair_loc] = results[22];
3160  input[current_idx_middle12_inner_pair_loc] = results[23];
3161  input[current_idx_middle3_pair_loc] = results[24];
3162  input[current_idx_middle3_inner_pair_loc] = results[25];
3163  input[current_idx_middle13_pair_loc] = results[26];
3164  input[current_idx_middle13_inner_pair_loc] = results[27];
3165  input[current_idx_middle23_pair_loc] = results[28];
3166  input[current_idx_middle23_inner_pair_loc] = results[29];
3167  input[current_idx_middle123_pair_loc] = results[30];
3168  input[current_idx_middle123_inner_pair_loc] = results[31];
3169  }
3170 }
3171 
3179 void apply_5qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
3180  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
3181 
3182  int index_step_inner = 1 << involved_qbits[0];
3183  int index_step_middle1 = 1 << involved_qbits[1];
3184  int index_step_middle2 = 1 << involved_qbits[2];
3185  int index_step_middle3 = 1 << involved_qbits[3];
3186  int index_step_outer = 1 << involved_qbits[4];
3187 
3188  int num_qubits = (int)std::log2(matrix_size);
3189  std::vector<int> is_target(num_qubits, 0);
3190  for (int q : involved_qbits) is_target[q] = 1;
3191 
3192  std::vector<int> non_targets;
3193  non_targets.reserve(num_qubits - 5);
3194  for (int q = 0; q < num_qubits; ++q) {
3195  if (!is_target[q]) non_targets.push_back(q);
3196  }
3197  tbb::parallel_for(
3198  tbb::blocked_range<int>(0, matrix_size >> 5),
3199  [&](const tbb::blocked_range<int>& range) {
3200  for (int block_idx = range.begin(); block_idx != range.end(); ++block_idx) {
3201  // Calculate base index properly
3202  int base = 0;
3203  for (size_t i = 0; i < non_targets.size(); ++i) {
3204  if (block_idx & (1ULL << i)) {
3205  base |= (1 << non_targets[i]);
3206  }
3207  }
3208  // Calculate all 32 indices using OR operations
3209  int current_idx_outer_loc = base;
3210  int current_idx_inner_loc = base | index_step_inner;
3211 
3212  int current_idx_middle1_loc = base | index_step_middle1;
3213  int current_idx_middle1_inner_loc = base | index_step_middle1 | index_step_inner;
3214 
3215  int current_idx_middle2_loc = base | index_step_middle2;
3216  int current_idx_middle2_inner_loc = base | index_step_middle2 | index_step_inner;
3217 
3218  int current_idx_middle12_loc = base | index_step_middle1 | index_step_middle2;
3219  int current_idx_middle12_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_inner;
3220 
3221  int current_idx_middle3_loc = base | index_step_middle3;
3222  int current_idx_middle3_inner_loc = base | index_step_middle3 | index_step_inner;
3223 
3224  int current_idx_middle13_loc = base | index_step_middle1 | index_step_middle3;
3225  int current_idx_middle13_inner_loc = base | index_step_middle1 | index_step_middle3 | index_step_inner;
3226 
3227  int current_idx_middle23_loc = base | index_step_middle2 | index_step_middle3;
3228  int current_idx_middle23_inner_loc = base | index_step_middle2 | index_step_middle3 | index_step_inner;
3229 
3230  int current_idx_middle123_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3;
3231  int current_idx_middle123_inner_loc = base | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
3232 
3233  int current_idx_outer_pair_loc = base | index_step_outer;
3234  int current_idx_inner_pair_loc = base | index_step_outer | index_step_inner;
3235 
3236  int current_idx_middle1_pair_loc = base | index_step_outer | index_step_middle1;
3237  int current_idx_middle1_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_inner;
3238 
3239  int current_idx_middle2_pair_loc = base | index_step_outer | index_step_middle2;
3240  int current_idx_middle2_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_inner;
3241 
3242  int current_idx_middle12_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2;
3243  int current_idx_middle12_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_inner;
3244 
3245  int current_idx_middle3_pair_loc = base | index_step_outer | index_step_middle3;
3246  int current_idx_middle3_inner_pair_loc = base | index_step_outer | index_step_middle3 | index_step_inner;
3247 
3248  int current_idx_middle13_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3;
3249  int current_idx_middle13_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle3 | index_step_inner;
3250 
3251  int current_idx_middle23_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3;
3252  int current_idx_middle23_inner_pair_loc = base | index_step_outer | index_step_middle2 | index_step_middle3 | index_step_inner;
3253 
3254  int current_idx_middle123_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3;
3255  int current_idx_middle123_inner_pair_loc = base | index_step_outer | index_step_middle1 | index_step_middle2 | index_step_middle3 | index_step_inner;
3256 
3257  __m256d element_00000_vec_real = _mm256_setzero_pd();
3258  __m256d element_00010_vec_real = _mm256_setzero_pd();
3259  __m256d element_00100_vec_real = _mm256_setzero_pd();
3260  __m256d element_00110_vec_real = _mm256_setzero_pd();
3261  __m256d element_01000_vec_real = _mm256_setzero_pd();
3262  __m256d element_01010_vec_real = _mm256_setzero_pd();
3263  __m256d element_01100_vec_real = _mm256_setzero_pd();
3264  __m256d element_01110_vec_real = _mm256_setzero_pd();
3265  __m256d element_10000_vec_real = _mm256_setzero_pd();
3266  __m256d element_10010_vec_real = _mm256_setzero_pd();
3267  __m256d element_10100_vec_real = _mm256_setzero_pd();
3268  __m256d element_10110_vec_real = _mm256_setzero_pd();
3269  __m256d element_11000_vec_real = _mm256_setzero_pd();
3270  __m256d element_11010_vec_real = _mm256_setzero_pd();
3271  __m256d element_11100_vec_real = _mm256_setzero_pd();
3272  __m256d element_11110_vec_real = _mm256_setzero_pd();
3273 
3274  if (involved_qbits[0] == 0) {
3275  // Preload all 32 elements instead of the unitary kernel matrix
3276  element_00000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_loc);
3277  element_00010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_loc);
3278  element_00100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_loc);
3279  element_00110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_loc);
3280  element_01000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_loc);
3281  element_01010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_loc);
3282  element_01100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_loc);
3283  element_01110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_loc);
3284  element_10000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_outer_pair_loc);
3285  element_10010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle1_pair_loc);
3286  element_10100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle2_pair_loc);
3287  element_10110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle12_pair_loc);
3288  element_11000_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle3_pair_loc);
3289  element_11010_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle13_pair_loc);
3290  element_11100_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle23_pair_loc);
3291  element_11110_vec_real = _mm256_loadu_pd((double*)input.get_data() + 2 * current_idx_middle123_pair_loc);
3292 
3293  }
3294  else {
3295  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer_loc;
3296  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner_loc;
3297 
3298  double* element_middle1 = (double*)input.get_data() + 2 * current_idx_middle1_loc;
3299  double* element_middle1_inner = (double*)input.get_data() + 2 * current_idx_middle1_inner_loc;
3300 
3301  double* element_middle2 = (double*)input.get_data() + 2 * current_idx_middle2_loc;
3302  double* element_middle2_inner = (double*)input.get_data() + 2 * current_idx_middle2_inner_loc;
3303 
3304  double* element_middle12 = (double*)input.get_data() + 2 * current_idx_middle12_loc;
3305  double* element_middle12_inner = (double*)input.get_data() + 2 * current_idx_middle12_inner_loc;
3306 
3307  double* element_middle3 = (double*)input.get_data() + 2 * current_idx_middle3_loc;
3308  double* element_middle3_inner = (double*)input.get_data() + 2 * current_idx_middle3_inner_loc;
3309 
3310  double* element_middle13 = (double*)input.get_data() + 2 * current_idx_middle13_loc;
3311  double* element_middle13_inner = (double*)input.get_data() + 2 * current_idx_middle13_inner_loc;
3312 
3313  double* element_middle23 = (double*)input.get_data() + 2 * current_idx_middle23_loc;
3314  double* element_middle23_inner = (double*)input.get_data() + 2 * current_idx_middle23_inner_loc;
3315 
3316  double* element_middle123 = (double*)input.get_data() + 2 * current_idx_middle123_loc;
3317  double* element_middle123_inner = (double*)input.get_data() + 2 * current_idx_middle123_inner_loc;
3318 
3319  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair_loc;
3320  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair_loc;
3321 
3322  double* element_middle1_pair = (double*)input.get_data() + 2 * current_idx_middle1_pair_loc;
3323  double* element_middle1_inner_pair = (double*)input.get_data() + 2 * current_idx_middle1_inner_pair_loc;
3324 
3325  double* element_middle2_pair = (double*)input.get_data() + 2 * current_idx_middle2_pair_loc;
3326  double* element_middle2_inner_pair = (double*)input.get_data() + 2 * current_idx_middle2_inner_pair_loc;
3327 
3328  double* element_middle12_pair = (double*)input.get_data() + 2 * current_idx_middle12_pair_loc;
3329  double* element_middle12_inner_pair = (double*)input.get_data() + 2 * current_idx_middle12_inner_pair_loc;
3330 
3331  double* element_middle3_pair = (double*)input.get_data() + 2 * current_idx_middle3_pair_loc;
3332  double* element_middle3_inner_pair = (double*)input.get_data() + 2 * current_idx_middle3_inner_pair_loc;
3333 
3334  double* element_middle13_pair = (double*)input.get_data() + 2 * current_idx_middle13_pair_loc;
3335  double* element_middle13_inner_pair = (double*)input.get_data() + 2 * current_idx_middle13_inner_pair_loc;
3336 
3337  double* element_middle23_pair = (double*)input.get_data() + 2 * current_idx_middle23_pair_loc;
3338  double* element_middle23_inner_pair = (double*)input.get_data() + 2 * current_idx_middle23_inner_pair_loc;
3339 
3340  double* element_middle123_pair = (double*)input.get_data() + 2 * current_idx_middle123_pair_loc;
3341  double* element_middle123_inner_pair = (double*)input.get_data() + 2 * current_idx_middle123_inner_pair_loc;
3342 
3343  element_00000_vec_real = get_AVX_vector(element_outer, element_inner);
3344  element_00010_vec_real = get_AVX_vector(element_middle1, element_middle1_inner);
3345  element_00100_vec_real = get_AVX_vector(element_middle2, element_middle2_inner);
3346  element_00110_vec_real = get_AVX_vector(element_middle12, element_middle12_inner);
3347  element_01000_vec_real = get_AVX_vector(element_middle3, element_middle3_inner);
3348  element_01010_vec_real = get_AVX_vector(element_middle13, element_middle13_inner);
3349  element_01100_vec_real = get_AVX_vector(element_middle23, element_middle23_inner);
3350  element_01110_vec_real = get_AVX_vector(element_middle123, element_middle123_inner);
3351  element_10000_vec_real = get_AVX_vector(element_outer_pair, element_inner_pair);
3352  element_10010_vec_real = get_AVX_vector(element_middle1_pair, element_middle1_inner_pair);
3353  element_10100_vec_real = get_AVX_vector(element_middle2_pair, element_middle2_inner_pair);
3354  element_10110_vec_real = get_AVX_vector(element_middle12_pair, element_middle12_inner_pair);
3355  element_11000_vec_real = get_AVX_vector(element_middle3_pair, element_middle3_inner_pair);
3356  element_11010_vec_real = get_AVX_vector(element_middle13_pair, element_middle13_inner_pair);
3357  element_11100_vec_real = get_AVX_vector(element_middle23_pair, element_middle23_inner_pair);
3358  element_11110_vec_real = get_AVX_vector(element_middle123_pair, element_middle123_inner_pair);
3359 
3360  }
3361  __m256d element_00000_vec_imag = _mm256_permute4x64_pd(element_00000_vec_real, 0b10110001);
3362  __m256d element_00010_vec_imag = _mm256_permute4x64_pd(element_00010_vec_real, 0b10110001);
3363  __m256d element_00100_vec_imag = _mm256_permute4x64_pd(element_00100_vec_real, 0b10110001);
3364  __m256d element_00110_vec_imag = _mm256_permute4x64_pd(element_00110_vec_real, 0b10110001);
3365  __m256d element_01000_vec_imag = _mm256_permute4x64_pd(element_01000_vec_real, 0b10110001);
3366  __m256d element_01010_vec_imag = _mm256_permute4x64_pd(element_01010_vec_real, 0b10110001);
3367  __m256d element_01100_vec_imag = _mm256_permute4x64_pd(element_01100_vec_real, 0b10110001);
3368  __m256d element_01110_vec_imag = _mm256_permute4x64_pd(element_01110_vec_real, 0b10110001);
3369  __m256d element_10000_vec_imag = _mm256_permute4x64_pd(element_10000_vec_real, 0b10110001);
3370  __m256d element_10010_vec_imag = _mm256_permute4x64_pd(element_10010_vec_real, 0b10110001);
3371  __m256d element_10100_vec_imag = _mm256_permute4x64_pd(element_10100_vec_real, 0b10110001);
3372  __m256d element_10110_vec_imag = _mm256_permute4x64_pd(element_10110_vec_real, 0b10110001);
3373  __m256d element_11000_vec_imag = _mm256_permute4x64_pd(element_11000_vec_real, 0b10110001);
3374  __m256d element_11010_vec_imag = _mm256_permute4x64_pd(element_11010_vec_real, 0b10110001);
3375  __m256d element_11100_vec_imag = _mm256_permute4x64_pd(element_11100_vec_real, 0b10110001);
3376  __m256d element_11110_vec_imag = _mm256_permute4x64_pd(element_11110_vec_real, 0b10110001);
3377 
3378  element_00000_vec_real = _mm256_mul_pd(element_00000_vec_real,neg);
3379  element_00010_vec_real = _mm256_mul_pd(element_00010_vec_real,neg);
3380  element_00100_vec_real = _mm256_mul_pd(element_00100_vec_real,neg);
3381  element_00110_vec_real = _mm256_mul_pd(element_00110_vec_real,neg);
3382  element_01000_vec_real = _mm256_mul_pd(element_01000_vec_real,neg);
3383  element_01010_vec_real = _mm256_mul_pd(element_01010_vec_real,neg);
3384  element_01100_vec_real = _mm256_mul_pd(element_01100_vec_real,neg);
3385  element_01110_vec_real = _mm256_mul_pd(element_01110_vec_real,neg);
3386  element_10000_vec_real = _mm256_mul_pd(element_10000_vec_real,neg);
3387  element_10010_vec_real = _mm256_mul_pd(element_10010_vec_real,neg);
3388  element_10100_vec_real = _mm256_mul_pd(element_10100_vec_real,neg);
3389  element_10110_vec_real = _mm256_mul_pd(element_10110_vec_real,neg);
3390  element_11000_vec_real = _mm256_mul_pd(element_11000_vec_real,neg);
3391  element_11010_vec_real = _mm256_mul_pd(element_11010_vec_real,neg);
3392  element_11100_vec_real = _mm256_mul_pd(element_11100_vec_real,neg);
3393  element_11110_vec_real = _mm256_mul_pd(element_11110_vec_real,neg);
3394 
3395  QGD_Complex16 results[32];
3396  for (int mult_idx = 0; mult_idx < 32; mult_idx++) {
3397  double* unitary_row_1 = (double*)unitary.get_data() + 64*mult_idx;
3398  double* unitary_row_2 = unitary_row_1 + 4;
3399  double* unitary_row_3 = unitary_row_1 + 8;
3400  double* unitary_row_4 = unitary_row_1 + 12;
3401  double* unitary_row_5 = unitary_row_1 + 16;
3402  double* unitary_row_6 = unitary_row_1 + 20;
3403  double* unitary_row_7 = unitary_row_1 + 24;
3404  double* unitary_row_8 = unitary_row_1 + 28;
3405  double* unitary_row_9 = unitary_row_1 + 32;
3406  double* unitary_row_10 = unitary_row_1 + 36;
3407  double* unitary_row_11 = unitary_row_1 + 40;
3408  double* unitary_row_12 = unitary_row_1 + 44;
3409  double* unitary_row_13 = unitary_row_1 + 48;
3410  double* unitary_row_14 = unitary_row_1 + 52;
3411  double* unitary_row_15 = unitary_row_1 + 56;
3412  double* unitary_row_16 = unitary_row_1 + 60;
3413 
3414  __m256d row1_vec = _mm256_loadu_pd(unitary_row_1);
3415  __m256d row2_vec = _mm256_loadu_pd(unitary_row_2);
3416  __m256d row3_vec = _mm256_loadu_pd(unitary_row_3);
3417  __m256d row4_vec = _mm256_loadu_pd(unitary_row_4);
3418  __m256d row5_vec = _mm256_loadu_pd(unitary_row_5);
3419  __m256d row6_vec = _mm256_loadu_pd(unitary_row_6);
3420  __m256d row7_vec = _mm256_loadu_pd(unitary_row_7);
3421  __m256d row8_vec = _mm256_loadu_pd(unitary_row_8);
3422  __m256d row9_vec = _mm256_loadu_pd(unitary_row_9);
3423  __m256d row10_vec = _mm256_loadu_pd(unitary_row_10);
3424  __m256d row11_vec = _mm256_loadu_pd(unitary_row_11);
3425  __m256d row12_vec = _mm256_loadu_pd(unitary_row_12);
3426  __m256d row13_vec = _mm256_loadu_pd(unitary_row_13);
3427  __m256d row14_vec = _mm256_loadu_pd(unitary_row_14);
3428  __m256d row15_vec = _mm256_loadu_pd(unitary_row_15);
3429  __m256d row16_vec = _mm256_loadu_pd(unitary_row_16);
3430 
3431  __m256d data_real = _mm256_setzero_pd();
3432  __m256d data_imag = _mm256_setzero_pd();
3433 
3434  data_real = _mm256_fmadd_pd(element_00000_vec_real, row1_vec, data_real);
3435  data_imag = _mm256_fmadd_pd(element_00000_vec_imag, row1_vec, data_imag);
3436  data_real = _mm256_fmadd_pd(element_00010_vec_real, row2_vec, data_real);
3437  data_imag = _mm256_fmadd_pd(element_00010_vec_imag, row2_vec, data_imag);
3438  data_real = _mm256_fmadd_pd(element_00100_vec_real, row3_vec, data_real);
3439  data_imag = _mm256_fmadd_pd(element_00100_vec_imag, row3_vec, data_imag);
3440  data_real = _mm256_fmadd_pd(element_00110_vec_real, row4_vec, data_real);
3441  data_imag = _mm256_fmadd_pd(element_00110_vec_imag, row4_vec, data_imag);
3442  data_real = _mm256_fmadd_pd(element_01000_vec_real, row5_vec, data_real);
3443  data_imag = _mm256_fmadd_pd(element_01000_vec_imag, row5_vec, data_imag);
3444  data_real = _mm256_fmadd_pd(element_01010_vec_real, row6_vec, data_real);
3445  data_imag = _mm256_fmadd_pd(element_01010_vec_imag, row6_vec, data_imag);
3446  data_real = _mm256_fmadd_pd(element_01100_vec_real, row7_vec, data_real);
3447  data_imag = _mm256_fmadd_pd(element_01100_vec_imag, row7_vec, data_imag);
3448  data_real = _mm256_fmadd_pd(element_01110_vec_real, row8_vec, data_real);
3449  data_imag = _mm256_fmadd_pd(element_01110_vec_imag, row8_vec, data_imag);
3450  data_real = _mm256_fmadd_pd(element_10000_vec_real, row9_vec, data_real);
3451  data_imag = _mm256_fmadd_pd(element_10000_vec_imag, row9_vec, data_imag);
3452  data_real = _mm256_fmadd_pd(element_10010_vec_real, row10_vec, data_real);
3453  data_imag = _mm256_fmadd_pd(element_10010_vec_imag, row10_vec, data_imag);
3454  data_real = _mm256_fmadd_pd(element_10100_vec_real, row11_vec, data_real);
3455  data_imag = _mm256_fmadd_pd(element_10100_vec_imag, row11_vec, data_imag);
3456  data_real = _mm256_fmadd_pd(element_10110_vec_real, row12_vec, data_real);
3457  data_imag = _mm256_fmadd_pd(element_10110_vec_imag, row12_vec, data_imag);
3458  data_real = _mm256_fmadd_pd(element_11000_vec_real, row13_vec, data_real);
3459  data_imag = _mm256_fmadd_pd(element_11000_vec_imag, row13_vec, data_imag);
3460  data_real = _mm256_fmadd_pd(element_11010_vec_real, row14_vec, data_real);
3461  data_imag = _mm256_fmadd_pd(element_11010_vec_imag, row14_vec, data_imag);
3462  data_real = _mm256_fmadd_pd(element_11100_vec_real, row15_vec, data_real);
3463  data_imag = _mm256_fmadd_pd(element_11100_vec_imag, row15_vec, data_imag);
3464  data_real = _mm256_fmadd_pd(element_11110_vec_real, row16_vec, data_real);
3465  data_imag = _mm256_fmadd_pd(element_11110_vec_imag, row16_vec, data_imag);
3466 
3467  __m256d final_vec = _mm256_hadd_pd(data_real, data_imag);
3468  final_vec = _mm256_permute4x64_pd(final_vec, 0b11011000);
3469  final_vec = _mm256_hadd_pd(final_vec, final_vec);
3470  __m128d low128 = _mm256_castpd256_pd128(final_vec);
3471  __m128d high128 = _mm256_extractf128_pd(final_vec, 1);
3472  results[mult_idx].real = _mm_cvtsd_f64(low128);
3473  results[mult_idx].imag = _mm_cvtsd_f64(high128);
3474  }
3475  input[current_idx_outer_loc] = results[0];
3476  input[current_idx_inner_loc] = results[1];
3477  input[current_idx_middle1_loc] = results[2];
3478  input[current_idx_middle1_inner_loc] = results[3];
3479  input[current_idx_middle2_loc] = results[4];
3480  input[current_idx_middle2_inner_loc] = results[5];
3481  input[current_idx_middle12_loc] = results[6];
3482  input[current_idx_middle12_inner_loc] = results[7];
3483  input[current_idx_middle3_loc] = results[8];
3484  input[current_idx_middle3_inner_loc] = results[9];
3485  input[current_idx_middle13_loc] = results[10];
3486  input[current_idx_middle13_inner_loc] = results[11];
3487  input[current_idx_middle23_loc] = results[12];
3488  input[current_idx_middle23_inner_loc] = results[13];
3489  input[current_idx_middle123_loc] = results[14];
3490  input[current_idx_middle123_inner_loc] = results[15];
3491  input[current_idx_outer_pair_loc] = results[16];
3492  input[current_idx_inner_pair_loc] = results[17];
3493  input[current_idx_middle1_pair_loc] = results[18];
3494  input[current_idx_middle1_inner_pair_loc] = results[19];
3495  input[current_idx_middle2_pair_loc] = results[20];
3496  input[current_idx_middle2_inner_pair_loc] = results[21];
3497  input[current_idx_middle12_pair_loc] = results[22];
3498  input[current_idx_middle12_inner_pair_loc] = results[23];
3499  input[current_idx_middle3_pair_loc] = results[24];
3500  input[current_idx_middle3_inner_pair_loc] = results[25];
3501  input[current_idx_middle13_pair_loc] = results[26];
3502  input[current_idx_middle13_inner_pair_loc] = results[27];
3503  input[current_idx_middle23_pair_loc] = results[28];
3504  input[current_idx_middle23_inner_pair_loc] = results[29];
3505  input[current_idx_middle123_pair_loc] = results[30];
3506  input[current_idx_middle123_inner_pair_loc] = results[31];
3507  }
3508 }
3509  );
3510 }
3511 
3512 
3520 void apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP(Matrix& two_qbit_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size){
3521  int inner_qbit = involved_qbits[0];
3522  int outer_qbit = involved_qbits[1];
3523  int index_step_outer = 1 << outer_qbit;
3524  int index_step_inner = 1 << inner_qbit;
3525  int current_idx = 0;
3526  __m256d neg = _mm256_setr_pd(1.0, -1.0, 1.0, -1.0);
3527  for (int current_idx_pair_outer=current_idx + index_step_outer; current_idx_pair_outer<input.rows; current_idx_pair_outer=current_idx_pair_outer+(index_step_outer << 1)){
3528 
3529  for (int current_idx_inner = 0; current_idx_inner < index_step_outer; current_idx_inner=current_idx_inner+(index_step_inner<<1)){
3530  for (int idx=0; idx<index_step_inner; idx++){
3531 
3532  int current_idx_outer_loc = current_idx + current_idx_inner + idx;
3533  int current_idx_inner_loc = current_idx + current_idx_inner + idx + index_step_inner;
3534  int current_idx_outer_pair_loc = current_idx_pair_outer + idx + current_idx_inner;
3535  int current_idx_inner_pair_loc = current_idx_pair_outer + idx + current_idx_inner + index_step_inner;
3536 
3537  int row_offset_outer = current_idx_outer_loc*input.cols;
3538  int row_offset_inner = current_idx_inner_loc*input.cols;
3539  int row_offset_outer_pair = current_idx_outer_pair_loc*input.cols;
3540  int row_offset_inner_pair = current_idx_inner_pair_loc*input.cols;
3541  #pragma omp parallel for
3542  for (int col_idx=0; col_idx<input.cols;col_idx++){
3543 
3544  int current_idx_outer = row_offset_outer + col_idx;
3545  int current_idx_inner = row_offset_inner + col_idx;
3546  int current_idx_outer_pair = row_offset_outer_pair + col_idx;
3547  int current_idx_inner_pair = row_offset_inner_pair + col_idx;
3548 
3549  double results[8] = {0.,0.,0.,0.,0.,0.,0.,0.};
3550 
3551  double* element_outer = (double*)input.get_data() + 2 * current_idx_outer;
3552  double* element_inner = (double*)input.get_data() + 2 * current_idx_inner;
3553  double* element_outer_pair = (double*)input.get_data() + 2 * current_idx_outer_pair;
3554  double* element_inner_pair = (double*)input.get_data() + 2 * current_idx_inner_pair;
3555 
3556  __m256d element_outer_vec = _mm256_loadu_pd(element_outer);
3557  element_outer_vec = _mm256_permute4x64_pd(element_outer_vec,0b11011000);
3558  __m256d element_inner_vec = _mm256_loadu_pd(element_inner);
3559  element_inner_vec = _mm256_permute4x64_pd(element_inner_vec,0b11011000);
3560  __m256d outer_inner_vec = _mm256_shuffle_pd(element_outer_vec,element_inner_vec,0b0000);
3561  outer_inner_vec = _mm256_permute4x64_pd(outer_inner_vec,0b11011000);
3562 
3563 
3564  __m256d element_outer_pair_vec = _mm256_loadu_pd(element_outer_pair);
3565  element_outer_pair_vec = _mm256_permute4x64_pd(element_outer_pair_vec,0b11011000);
3566  __m256d element_inner_pair_vec = _mm256_loadu_pd(element_inner_pair);
3567  element_inner_pair_vec = _mm256_permute4x64_pd(element_inner_pair_vec,0b11011000);
3568  __m256d outer_inner_pair_vec = _mm256_shuffle_pd(element_outer_pair_vec,element_inner_pair_vec,0b0000);
3569  outer_inner_pair_vec = _mm256_permute4x64_pd(outer_inner_pair_vec,0b11011000);
3570 
3571 
3572 
3573 
3574  for (int mult_idx=0; mult_idx<4; mult_idx++){
3575  double* unitary_row_01 = (double*)two_qbit_unitary.get_data() + 8*mult_idx;
3576  double* unitary_row_23 = (double*)two_qbit_unitary.get_data() + 8*mult_idx + 4;
3577 
3578  __m256d unitary_row_01_vec = _mm256_loadu_pd(unitary_row_01);
3579  __m256d unitary_row_23_vec = _mm256_loadu_pd(unitary_row_23);
3580 
3581  __m256d result_upper_vec = complex_mult_AVX(outer_inner_vec,unitary_row_01_vec,neg);
3582 
3583  __m256d result_lower_vec = complex_mult_AVX(outer_inner_pair_vec,unitary_row_23_vec,neg);
3584 
3585  __m256d result_vec = _mm256_hadd_pd(result_upper_vec,result_lower_vec);
3586  result_vec = _mm256_hadd_pd(result_vec,result_vec);
3587  double* result = (double*)&result_vec;
3588  results[mult_idx*2] = result[0];
3589  results[mult_idx*2+1] = result[2];
3590  }
3591  input[current_idx_outer].real = results[0];
3592  input[current_idx_outer].imag = results[1];
3593  input[current_idx_inner].real = results[2];
3594  input[current_idx_inner].imag = results[3];
3595  input[current_idx_outer_pair].real = results[4];
3596  input[current_idx_outer_pair].imag = results[5];
3597  input[current_idx_inner_pair].real = results[6];
3598  input[current_idx_inner_pair].imag = results[7];
3599  }
3600  }
3601 
3602  }
3603  current_idx = current_idx + (index_step_outer << 1);
3604  }
3605 }
3606 
3616 void
3617 apply_crot_kernel_to_matrix_input_AVX(Matrix& u3_1qbit1, Matrix& u3_1qbit2, Matrix& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
3618  input.ensure_aligned();
3619 
3620  int index_step_target = 1 << target_qbit;
3621  int current_idx = 0;
3622 
3623  // load elements of the first U3 unitary into 256bit registers (8 registers)
3624  __m256d u3_1bit_00r_vec = _mm256_broadcast_sd(&u3_1qbit1[0].real);
3625  __m256d u3_1bit_00i_vec = _mm256_broadcast_sd(&u3_1qbit1[0].imag);
3626  __m256d u3_1bit_01r_vec = _mm256_broadcast_sd(&u3_1qbit1[1].real);
3627  __m256d u3_1bit_01i_vec = _mm256_broadcast_sd(&u3_1qbit1[1].imag);
3628  __m256d u3_1bit_10r_vec = _mm256_broadcast_sd(&u3_1qbit1[2].real);
3629  __m256d u3_1bit_10i_vec = _mm256_broadcast_sd(&u3_1qbit1[2].imag);
3630  __m256d u3_1bit_11r_vec = _mm256_broadcast_sd(&u3_1qbit1[3].real);
3631  __m256d u3_1bit_11i_vec = _mm256_broadcast_sd(&u3_1qbit1[3].imag);
3632  // load elements of the second U3 unitary into 256bit registers (8 registers)
3633  __m256d u3_1bit2_00r_vec = _mm256_broadcast_sd(&u3_1qbit2[0].real);
3634  __m256d u3_1bit2_00i_vec = _mm256_broadcast_sd(&u3_1qbit2[0].imag);
3635  __m256d u3_1bit2_01r_vec = _mm256_broadcast_sd(&u3_1qbit2[1].real);
3636  __m256d u3_1bit2_01i_vec = _mm256_broadcast_sd(&u3_1qbit2[1].imag);
3637  __m256d u3_1bit2_10r_vec = _mm256_broadcast_sd(&u3_1qbit2[2].real);
3638  __m256d u3_1bit2_10i_vec = _mm256_broadcast_sd(&u3_1qbit2[2].imag);
3639  __m256d u3_1bit2_11r_vec = _mm256_broadcast_sd(&u3_1qbit2[3].real);
3640  __m256d u3_1bit2_11i_vec = _mm256_broadcast_sd(&u3_1qbit2[3].imag);
3641 
3642 
3643  for ( int current_idx_pair=current_idx + index_step_target; current_idx_pair<matrix_size; current_idx_pair=current_idx_pair+(index_step_target << 1) ) {
3644 
3645 
3646  for (int idx = 0; idx < index_step_target; idx++) {
3647 
3648 
3649  int current_idx_loc = current_idx + idx;
3650  int current_idx_pair_loc = current_idx_pair + idx;
3651 
3652  int row_offset = current_idx_loc * input.stride;
3653  int row_offset_pair = current_idx_pair_loc * input.stride;
3654  for (int col_idx = 0; col_idx < 2 * (input.cols - 3); col_idx = col_idx + 8) {
3655  double* element = (double*)input.get_data() + 2 * row_offset;
3656  double* element_pair = (double*)input.get_data() + 2 * row_offset_pair;
3657  if ((current_idx_loc >> control_qbit) & 1) {
3658 
3659 
3660  // extract successive elements from arrays element, element_pair
3661  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
3662  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
3663  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
3664  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
3665  element_vec = tmp;
3666 
3667  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
3668  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
3669  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
3670  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
3671  element_pair_vec = tmp;
3672 
3673  __m256d vec3 = _mm256_mul_pd(u3_1bit_00r_vec, element_vec);
3674  vec3 = _mm256_fnmadd_pd(u3_1bit_00i_vec, element_vec2, vec3);
3675  __m256d vec4 = _mm256_mul_pd(u3_1bit_01r_vec, element_pair_vec);
3676  vec4 = _mm256_fnmadd_pd(u3_1bit_01i_vec, element_pair_vec2, vec4);
3677  vec3 = _mm256_add_pd(vec3, vec4);
3678  __m256d vec5 = _mm256_mul_pd(u3_1bit_00r_vec, element_vec2);
3679  vec5 = _mm256_fmadd_pd(u3_1bit_00i_vec, element_vec, vec5);
3680  __m256d vec6 = _mm256_mul_pd(u3_1bit_01r_vec, element_pair_vec2);
3681  vec6 = _mm256_fmadd_pd(u3_1bit_01i_vec, element_pair_vec, vec6);
3682  vec5 = _mm256_add_pd(vec5, vec6);
3683 
3684  // 6 store the transformed elements in vec3
3685  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
3686  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
3687  vec3 = tmp;
3688  _mm256_storeu_pd(element + col_idx, vec3);
3689  _mm256_storeu_pd(element + col_idx + 4, vec5);
3690 
3691  __m256d vec7 = _mm256_mul_pd(u3_1bit_10r_vec, element_vec);
3692  vec7 = _mm256_fnmadd_pd(u3_1bit_10i_vec, element_vec2, vec7);
3693  __m256d vec8 = _mm256_mul_pd(u3_1bit_11r_vec, element_pair_vec);
3694  vec8 = _mm256_fnmadd_pd(u3_1bit_11i_vec, element_pair_vec2, vec8);
3695  vec7 = _mm256_add_pd(vec7, vec8);
3696  __m256d vec9 = _mm256_mul_pd(u3_1bit_10r_vec, element_vec2);
3697  vec9 = _mm256_fmadd_pd(u3_1bit_10i_vec, element_vec, vec9);
3698  __m256d vec10 = _mm256_mul_pd(u3_1bit_11r_vec, element_pair_vec2);
3699  vec10 = _mm256_fmadd_pd(u3_1bit_11i_vec, element_pair_vec, vec10);
3700  vec9 = _mm256_add_pd(vec9, vec10);
3701 
3702  // 6 store the transformed elements in vec3
3703  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
3704  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
3705  vec7 = tmp;
3706  _mm256_storeu_pd(element_pair + col_idx, vec7);
3707  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
3708 
3709 
3710 
3711  }
3712  else {
3713 
3714 
3715  // extract successive elements from arrays element, element_pair
3716  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
3717  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
3718  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
3719  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
3720  element_vec = tmp;
3721 
3722  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
3723  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
3724  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
3725  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
3726  element_pair_vec = tmp;
3727 
3728  __m256d vec3 = _mm256_mul_pd(u3_1bit2_00r_vec, element_vec);
3729  vec3 = _mm256_fnmadd_pd(u3_1bit2_00i_vec, element_vec2, vec3);
3730  __m256d vec4 = _mm256_mul_pd(u3_1bit2_01r_vec, element_pair_vec);
3731  vec4 = _mm256_fnmadd_pd(u3_1bit2_01i_vec, element_pair_vec2, vec4);
3732  vec3 = _mm256_add_pd(vec3, vec4);
3733  __m256d vec5 = _mm256_mul_pd(u3_1bit2_00r_vec, element_vec2);
3734  vec5 = _mm256_fmadd_pd(u3_1bit2_00i_vec, element_vec, vec5);
3735  __m256d vec6 = _mm256_mul_pd(u3_1bit2_01r_vec, element_pair_vec2);
3736  vec6 = _mm256_fmadd_pd(u3_1bit2_01i_vec, element_pair_vec, vec6);
3737  vec5 = _mm256_add_pd(vec5, vec6);
3738 
3739  // 6 store the transformed elements in vec3
3740  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
3741  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
3742  vec3 = tmp;
3743  _mm256_storeu_pd(element + col_idx, vec3);
3744  _mm256_storeu_pd(element + col_idx + 4, vec5);
3745 
3746  __m256d vec7 = _mm256_mul_pd(u3_1bit2_10r_vec, element_vec);
3747  vec7 = _mm256_fnmadd_pd(u3_1bit2_10i_vec, element_vec2, vec7);
3748  __m256d vec8 = _mm256_mul_pd(u3_1bit2_11r_vec, element_pair_vec);
3749  vec8 = _mm256_fnmadd_pd(u3_1bit2_11i_vec, element_pair_vec2, vec8);
3750  vec7 = _mm256_add_pd(vec7, vec8);
3751  __m256d vec9 = _mm256_mul_pd(u3_1bit2_10r_vec, element_vec2);
3752  vec9 = _mm256_fmadd_pd(u3_1bit2_10i_vec, element_vec, vec9);
3753  __m256d vec10 = _mm256_mul_pd(u3_1bit2_11r_vec, element_pair_vec2);
3754  vec10 = _mm256_fmadd_pd(u3_1bit2_11i_vec, element_pair_vec, vec10);
3755  vec9 = _mm256_add_pd(vec9, vec10);
3756 
3757  // 6 store the transformed elements in vec3
3758  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
3759  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
3760  vec7 = tmp;
3761  _mm256_storeu_pd(element_pair + col_idx, vec7);
3762  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
3763  }
3764  }
3765 
3766  int remainder = input.cols % 4;
3767  if (remainder != 0) {
3768 
3769  for (int col_idx = input.cols-remainder; col_idx < input.cols; col_idx++) {
3770  int index = row_offset+col_idx;
3771  int index_pair = row_offset_pair+col_idx;
3772  if ( (current_idx_loc >> control_qbit) & 1 ) {
3773 
3774 
3775 
3776  QGD_Complex16 element = input[index];
3777  QGD_Complex16 element_pair = input[index_pair];
3778 
3779  QGD_Complex16 tmp1 = mult(u3_1qbit1[0], element);
3780  QGD_Complex16 tmp2 = mult(u3_1qbit1[1], element_pair);
3781 
3782  input[index].real = tmp1.real + tmp2.real;
3783  input[index].imag = tmp1.imag + tmp2.imag;
3784 
3785  tmp1 = mult(u3_1qbit1[2], element);
3786  tmp2 = mult(u3_1qbit1[3], element_pair);
3787 
3788  input[index_pair].real = tmp1.real + tmp2.real;
3789  input[index_pair].imag = tmp1.imag + tmp2.imag;
3790 
3791  }
3792 
3793  else {
3794  QGD_Complex16 element = input[index];
3795  QGD_Complex16 element_pair = input[index_pair];
3796 
3797  QGD_Complex16 tmp1 = mult(u3_1qbit2[0], element);
3798  QGD_Complex16 tmp2 = mult(u3_1qbit2[1], element_pair);
3799 
3800  input[index].real = tmp1.real + tmp2.real;
3801  input[index].imag = tmp1.imag + tmp2.imag;
3802 
3803  tmp1 = mult(u3_1qbit2[2], element);
3804  tmp2 = mult(u3_1qbit2[3], element_pair);
3805 
3806  input[index_pair].real = tmp1.real + tmp2.real;
3807  input[index_pair].imag = tmp1.imag + tmp2.imag;
3808  }
3809  }
3810 
3811  }
3812  //std::cout << current_idx_target << " " << current_idx_target_pair << std::endl;
3813 
3814 
3815  }
3816 
3817 
3818 
3819  current_idx = current_idx + (index_step_target << 1);
3820 
3821  }
3822 
3823 
3824 }
3825 
3835 void
3836 apply_crot_kernel_to_matrix_input_AVX_parallel(Matrix& u3_1qbit1,Matrix& u3_1qbit2, Matrix& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
3837 
3838 
3839  input.ensure_aligned();
3840 
3841  int index_step_target = 1 << target_qbit;
3842 
3843  // load elements of the U3 unitary into 256bit registers (8 registers)
3844  __m256d u3_1bit_00r_vec = _mm256_broadcast_sd(&u3_1qbit1[0].real);
3845  __m256d u3_1bit_00i_vec = _mm256_broadcast_sd(&u3_1qbit1[0].imag);
3846  __m256d u3_1bit_01r_vec = _mm256_broadcast_sd(&u3_1qbit1[1].real);
3847  __m256d u3_1bit_01i_vec = _mm256_broadcast_sd(&u3_1qbit1[1].imag);
3848  __m256d u3_1bit_10r_vec = _mm256_broadcast_sd(&u3_1qbit1[2].real);
3849  __m256d u3_1bit_10i_vec = _mm256_broadcast_sd(&u3_1qbit1[2].imag);
3850  __m256d u3_1bit_11r_vec = _mm256_broadcast_sd(&u3_1qbit1[3].real);
3851  __m256d u3_1bit_11i_vec = _mm256_broadcast_sd(&u3_1qbit1[3].imag);
3852 
3853  __m256d u3_1bit2_00r_vec = _mm256_broadcast_sd(&u3_1qbit2[0].real);
3854  __m256d u3_1bit2_00i_vec = _mm256_broadcast_sd(&u3_1qbit2[0].imag);
3855  __m256d u3_1bit2_01r_vec = _mm256_broadcast_sd(&u3_1qbit2[1].real);
3856  __m256d u3_1bit2_01i_vec = _mm256_broadcast_sd(&u3_1qbit2[1].imag);
3857  __m256d u3_1bit2_10r_vec = _mm256_broadcast_sd(&u3_1qbit2[2].real);
3858  __m256d u3_1bit2_10i_vec = _mm256_broadcast_sd(&u3_1qbit2[2].imag);
3859  __m256d u3_1bit2_11r_vec = _mm256_broadcast_sd(&u3_1qbit2[3].real);
3860  __m256d u3_1bit2_11i_vec = _mm256_broadcast_sd(&u3_1qbit2[3].imag);
3861 
3862 
3863  int parallel_outer_cycles = matrix_size/(index_step_target << 1);
3864  int outer_grain_size;
3865  if ( index_step_target <= 2 ) {
3866  outer_grain_size = 32;
3867  }
3868  else if ( index_step_target <= 4 ) {
3869  outer_grain_size = 16;
3870  }
3871  else if ( index_step_target <= 8 ) {
3872  outer_grain_size = 8;
3873  }
3874  else if ( index_step_target <= 16 ) {
3875  outer_grain_size = 4;
3876  }
3877  else {
3878  outer_grain_size = 2;
3879  }
3880 
3881 
3882  tbb::parallel_for( tbb::blocked_range<int>(0,parallel_outer_cycles,outer_grain_size), [&](tbb::blocked_range<int> r) {
3883 
3884  int current_idx = r.begin()*(index_step_target << 1);
3885  int current_idx_pair = index_step_target + r.begin()*(index_step_target << 1);
3886 
3887  for (int rdx=r.begin(); rdx<r.end(); rdx++) {
3888 
3889 
3890  tbb::parallel_for( tbb::blocked_range<int>(0,index_step_target,32), [&](tbb::blocked_range<int> r) {
3891  for (int idx=r.begin(); idx<r.end(); ++idx) {
3892 
3893 
3894  int current_idx_loc = current_idx + idx;
3895  int current_idx_pair_loc = current_idx_pair + idx;
3896 
3897  int row_offset = current_idx_loc * input.stride;
3898  int row_offset_pair = current_idx_pair_loc * input.stride;
3899 
3900  if ((current_idx_loc >> control_qbit) & 1) {
3901 
3902 
3903  double* element = (double*)input.get_data() + 2 * row_offset;
3904  double* element_pair = (double*)input.get_data() + 2 * row_offset_pair;
3905 
3906 
3907  for (int col_idx = 0; col_idx < 2 * (input.cols - 3); col_idx = col_idx + 8) {
3908 
3909  // extract successive elements from arrays element, element_pair
3910  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
3911  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
3912  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
3913  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
3914  element_vec = tmp;
3915 
3916  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
3917  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
3918  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
3919  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
3920  element_pair_vec = tmp;
3921 
3922  __m256d vec3 = _mm256_mul_pd(u3_1bit_00r_vec, element_vec);
3923  vec3 = _mm256_fnmadd_pd(u3_1bit_00i_vec, element_vec2, vec3);
3924  __m256d vec4 = _mm256_mul_pd(u3_1bit_01r_vec, element_pair_vec);
3925  vec4 = _mm256_fnmadd_pd(u3_1bit_01i_vec, element_pair_vec2, vec4);
3926  vec3 = _mm256_add_pd(vec3, vec4);
3927  __m256d vec5 = _mm256_mul_pd(u3_1bit_00r_vec, element_vec2);
3928  vec5 = _mm256_fmadd_pd(u3_1bit_00i_vec, element_vec, vec5);
3929  __m256d vec6 = _mm256_mul_pd(u3_1bit_01r_vec, element_pair_vec2);
3930  vec6 = _mm256_fmadd_pd(u3_1bit_01i_vec, element_pair_vec, vec6);
3931  vec5 = _mm256_add_pd(vec5, vec6);
3932 
3933  // 6 store the transformed elements in vec3
3934  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
3935  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
3936  vec3 = tmp;
3937  _mm256_storeu_pd(element + col_idx, vec3);
3938  _mm256_storeu_pd(element + col_idx + 4, vec5);
3939 
3940  __m256d vec7 = _mm256_mul_pd(u3_1bit_10r_vec, element_vec);
3941  vec7 = _mm256_fnmadd_pd(u3_1bit_10i_vec, element_vec2, vec7);
3942  __m256d vec8 = _mm256_mul_pd(u3_1bit_11r_vec, element_pair_vec);
3943  vec8 = _mm256_fnmadd_pd(u3_1bit_11i_vec, element_pair_vec2, vec8);
3944  vec7 = _mm256_add_pd(vec7, vec8);
3945  __m256d vec9 = _mm256_mul_pd(u3_1bit_10r_vec, element_vec2);
3946  vec9 = _mm256_fmadd_pd(u3_1bit_10i_vec, element_vec, vec9);
3947  __m256d vec10 = _mm256_mul_pd(u3_1bit_11r_vec, element_pair_vec2);
3948  vec10 = _mm256_fmadd_pd(u3_1bit_11i_vec, element_pair_vec, vec10);
3949  vec9 = _mm256_add_pd(vec9, vec10);
3950 
3951  // 6 store the transformed elements in vec3
3952  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
3953  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
3954  vec7 = tmp;
3955  _mm256_storeu_pd(element_pair + col_idx, vec7);
3956  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
3957  }
3958 
3959  int remainder = input.cols % 4;
3960  if (remainder != 0) {
3961 
3962  for (int col_idx = input.cols-remainder; col_idx < input.cols; col_idx++) {
3963  int index = row_offset + col_idx;
3964  int index_pair = row_offset_pair + col_idx;
3965 
3966  QGD_Complex16 element = input[index];
3967  QGD_Complex16 element_pair = input[index_pair];
3968 
3969  QGD_Complex16 tmp1 = mult(u3_1qbit1[0], element);
3970  QGD_Complex16 tmp2 = mult(u3_1qbit1[1], element_pair);
3971 
3972  input[index].real = tmp1.real + tmp2.real;
3973  input[index].imag = tmp1.imag + tmp2.imag;
3974 
3975  tmp1 = mult(u3_1qbit1[2], element);
3976  tmp2 = mult(u3_1qbit1[3], element_pair);
3977 
3978  input[index_pair].real = tmp1.real + tmp2.real;
3979  input[index_pair].imag = tmp1.imag + tmp2.imag;
3980  }
3981 
3982  }
3983 
3984  }
3985 
3986  else {
3987 
3988  double* element = (double*)input.get_data() + 2 * row_offset;
3989  double* element_pair = (double*)input.get_data() + 2 * row_offset_pair;
3990 
3991 
3992  for (int col_idx = 0; col_idx < 2 * (input.cols - 3); col_idx = col_idx + 8) {
3993 
3994  // extract successive elements from arrays element, element_pair
3995  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
3996  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
3997  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
3998  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
3999  element_vec = tmp;
4000 
4001  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
4002  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
4003  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
4004  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
4005  element_pair_vec = tmp;
4006 
4007  __m256d vec3 = _mm256_mul_pd(u3_1bit2_00r_vec, element_vec);
4008  vec3 = _mm256_fnmadd_pd(u3_1bit2_00i_vec, element_vec2, vec3);
4009  __m256d vec4 = _mm256_mul_pd(u3_1bit2_01r_vec, element_pair_vec);
4010  vec4 = _mm256_fnmadd_pd(u3_1bit2_01i_vec, element_pair_vec2, vec4);
4011  vec3 = _mm256_add_pd(vec3, vec4);
4012  __m256d vec5 = _mm256_mul_pd(u3_1bit2_00r_vec, element_vec2);
4013  vec5 = _mm256_fmadd_pd(u3_1bit2_00i_vec, element_vec, vec5);
4014  __m256d vec6 = _mm256_mul_pd(u3_1bit2_01r_vec, element_pair_vec2);
4015  vec6 = _mm256_fmadd_pd(u3_1bit2_01i_vec, element_pair_vec, vec6);
4016  vec5 = _mm256_add_pd(vec5, vec6);
4017 
4018  // 6 store the transformed elements in vec3
4019  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
4020  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
4021  vec3 = tmp;
4022  _mm256_storeu_pd(element + col_idx, vec3);
4023  _mm256_storeu_pd(element + col_idx + 4, vec5);
4024 
4025  __m256d vec7 = _mm256_mul_pd(u3_1bit2_10r_vec, element_vec);
4026  vec7 = _mm256_fnmadd_pd(u3_1bit2_10i_vec, element_vec2, vec7);
4027  __m256d vec8 = _mm256_mul_pd(u3_1bit2_11r_vec, element_pair_vec);
4028  vec8 = _mm256_fnmadd_pd(u3_1bit2_11i_vec, element_pair_vec2, vec8);
4029  vec7 = _mm256_add_pd(vec7, vec8);
4030  __m256d vec9 = _mm256_mul_pd(u3_1bit2_10r_vec, element_vec2);
4031  vec9 = _mm256_fmadd_pd(u3_1bit2_10i_vec, element_vec, vec9);
4032  __m256d vec10 = _mm256_mul_pd(u3_1bit2_11r_vec, element_pair_vec2);
4033  vec10 = _mm256_fmadd_pd(u3_1bit2_11i_vec, element_pair_vec, vec10);
4034  vec9 = _mm256_add_pd(vec9, vec10);
4035 
4036  // 6 store the transformed elements in vec3
4037  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
4038  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
4039  vec7 = tmp;
4040  _mm256_storeu_pd(element_pair + col_idx, vec7);
4041  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
4042  }
4043 
4044  int remainder = input.cols % 4;
4045  if (remainder != 0) {
4046 
4047  for (int col_idx = input.cols-remainder; col_idx < input.cols; col_idx++) {
4048  int index = row_offset + col_idx;
4049  int index_pair = row_offset_pair + col_idx;
4050 
4051  QGD_Complex16 element = input[index];
4052  QGD_Complex16 element_pair = input[index_pair];
4053 
4054  QGD_Complex16 tmp1 = mult(u3_1qbit2[0], element);
4055  QGD_Complex16 tmp2 = mult(u3_1qbit2[1], element_pair);
4056 
4057  input[index].real = tmp1.real + tmp2.real;
4058  input[index].imag = tmp1.imag + tmp2.imag;
4059 
4060  tmp1 = mult(u3_1qbit2[2], element);
4061  tmp2 = mult(u3_1qbit2[3], element_pair);
4062 
4063  input[index_pair].real = tmp1.real + tmp2.real;
4064  input[index_pair].imag = tmp1.imag + tmp2.imag;
4065  }
4066 
4067  }
4068  }
4069 
4070 
4071  //std::cout << current_idx_target << " " << current_idx_target_pair << std::endl;
4072 
4073 
4074  }
4075  });
4076 
4077 
4078 
4079  current_idx = current_idx + (index_step_target << 1);
4080  current_idx_pair = current_idx_pair + (index_step_target << 1);
4081 
4082  }
4083  });
4084 
4085 
4086 }
4087 
4088 void
4089 apply_crot_kernel_to_matrix_input_from_right_AVX(Matrix& u3_1qbit1, Matrix& u3_1qbit2, Matrix& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
4090 
4091  input.ensure_aligned();
4092 
4093  const int index_step_target = 1 << target_qbit;
4094 
4095  auto select_gate = [&](int block_start) -> Matrix& {
4096  return ((control_qbit < 0) || ((block_start >> control_qbit) & 1)) ? u3_1qbit1 : u3_1qbit2;
4097  };
4098 
4099  auto apply_gate_block = [&](Matrix& gate, double* element, double* element_pair, int avx_limit, int& col_idx) {
4100  const __m256d u00r = _mm256_broadcast_sd(&gate[0].real);
4101  const __m256d u00i = _mm256_broadcast_sd(&gate[0].imag);
4102  const __m256d u01r = _mm256_broadcast_sd(&gate[1].real);
4103  const __m256d u01i = _mm256_broadcast_sd(&gate[1].imag);
4104  const __m256d u10r = _mm256_broadcast_sd(&gate[2].real);
4105  const __m256d u10i = _mm256_broadcast_sd(&gate[2].imag);
4106  const __m256d u11r = _mm256_broadcast_sd(&gate[3].real);
4107  const __m256d u11i = _mm256_broadcast_sd(&gate[3].imag);
4108 
4109  for (col_idx = 0; col_idx < avx_limit; col_idx += 8) {
4110  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
4111  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
4112  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
4113  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
4114  element_vec = tmp;
4115 
4116  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
4117  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
4118  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
4119  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
4120  element_pair_vec = tmp;
4121 
4122  __m256d vec3 = _mm256_mul_pd(u00r, element_vec);
4123  vec3 = _mm256_fnmadd_pd(u00i, element_vec2, vec3);
4124  __m256d vec4 = _mm256_mul_pd(u10r, element_pair_vec);
4125  vec4 = _mm256_fnmadd_pd(u10i, element_pair_vec2, vec4);
4126  vec3 = _mm256_add_pd(vec3, vec4);
4127  __m256d vec5 = _mm256_mul_pd(u00r, element_vec2);
4128  vec5 = _mm256_fmadd_pd(u00i, element_vec, vec5);
4129  __m256d vec6 = _mm256_mul_pd(u10r, element_pair_vec2);
4130  vec6 = _mm256_fmadd_pd(u10i, element_pair_vec, vec6);
4131  vec5 = _mm256_add_pd(vec5, vec6);
4132 
4133  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
4134  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
4135  vec3 = tmp;
4136  _mm256_storeu_pd(element + col_idx, vec3);
4137  _mm256_storeu_pd(element + col_idx + 4, vec5);
4138 
4139  __m256d vec7 = _mm256_mul_pd(u01r, element_vec);
4140  vec7 = _mm256_fnmadd_pd(u01i, element_vec2, vec7);
4141  __m256d vec8 = _mm256_mul_pd(u11r, element_pair_vec);
4142  vec8 = _mm256_fnmadd_pd(u11i, element_pair_vec2, vec8);
4143  vec7 = _mm256_add_pd(vec7, vec8);
4144  __m256d vec9 = _mm256_mul_pd(u01r, element_vec2);
4145  vec9 = _mm256_fmadd_pd(u01i, element_vec, vec9);
4146  __m256d vec10 = _mm256_mul_pd(u11r, element_pair_vec2);
4147  vec10 = _mm256_fmadd_pd(u11i, element_pair_vec, vec10);
4148  vec9 = _mm256_add_pd(vec9, vec10);
4149 
4150  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
4151  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
4152  vec7 = tmp;
4153  _mm256_storeu_pd(element_pair + col_idx, vec7);
4154  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
4155  }
4156  };
4157 
4158  for (int row_idx = 0; row_idx < input.rows; ++row_idx) {
4159  const int row_offset = row_idx * input.stride;
4160  int current_idx = 0;
4161  int current_idx_pair = index_step_target;
4162 
4163  while (current_idx_pair < input.cols) {
4164  const bool mixed = (control_qbit >= 0 && control_qbit < target_qbit);
4165 
4166  if (!mixed) {
4167  Matrix& gate = select_gate(current_idx);
4168  double* element = (double*)input.get_data() + 2 * (row_offset + current_idx);
4169  double* element_pair = (double*)input.get_data() + 2 * (row_offset + current_idx_pair);
4170  int col_idx = 0;
4171  const int avx_limit = 2 * (index_step_target - 3);
4172  apply_gate_block(gate, element, element_pair, avx_limit, col_idx);
4173 
4174  for (int c = col_idx / 2; c < index_step_target; ++c) {
4175  const int index = row_offset + current_idx + c;
4176  const int index_pair = row_offset + current_idx_pair + c;
4177  QGD_Complex16 a = input[index];
4178  QGD_Complex16 b = input[index_pair];
4179  QGD_Complex16 tmp1 = mult(gate[0], a);
4180  QGD_Complex16 tmp2 = mult(gate[2], b);
4181  input[index].real = tmp1.real + tmp2.real;
4182  input[index].imag = tmp1.imag + tmp2.imag;
4183  tmp1 = mult(gate[1], a);
4184  tmp2 = mult(gate[3], b);
4185  input[index_pair].real = tmp1.real + tmp2.real;
4186  input[index_pair].imag = tmp1.imag + tmp2.imag;
4187  }
4188  } else {
4189  for (int idx = 0; idx < index_step_target; ++idx) {
4190  const int current_idx_loc = current_idx + idx;
4191  const int current_idx_pair_loc = current_idx_pair + idx;
4192  Matrix& gate = select_gate(current_idx_loc);
4193  const int index = row_offset + current_idx_loc;
4194  const int index_pair = row_offset + current_idx_pair_loc;
4195  QGD_Complex16 a = input[index];
4196  QGD_Complex16 b = input[index_pair];
4197  QGD_Complex16 tmp1 = mult(gate[0], a);
4198  QGD_Complex16 tmp2 = mult(gate[2], b);
4199  input[index].real = tmp1.real + tmp2.real;
4200  input[index].imag = tmp1.imag + tmp2.imag;
4201  tmp1 = mult(gate[1], a);
4202  tmp2 = mult(gate[3], b);
4203  input[index_pair].real = tmp1.real + tmp2.real;
4204  input[index_pair].imag = tmp1.imag + tmp2.imag;
4205  }
4206  }
4207 
4208  current_idx += (index_step_target << 1);
4209  current_idx_pair += (index_step_target << 1);
4210  }
4211  }
4212 
4213  (void)matrix_size;
4214 }
4215 
4216 void
4217 apply_crot_kernel_to_matrix_input_from_right_AVX_parallel(Matrix& u3_1qbit1,Matrix& u3_1qbit2, Matrix& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
4218 
4219  input.ensure_aligned();
4220 
4221  const int index_step_target = 1 << target_qbit;
4222 
4223  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
4224  for (int row_idx = range.begin(); row_idx < range.end(); ++row_idx) {
4225  const int row_offset = row_idx * input.stride;
4226  int current_idx = 0;
4227  int current_idx_pair = index_step_target;
4228 
4229  while (current_idx_pair < input.cols) {
4230  const bool mixed = (control_qbit >= 0 && control_qbit < target_qbit);
4231 
4232  if (!mixed) {
4233  Matrix& gate = ((control_qbit < 0) || ((current_idx >> control_qbit) & 1)) ? u3_1qbit1 : u3_1qbit2;
4234  const __m256d u00r = _mm256_broadcast_sd(&gate[0].real);
4235  const __m256d u00i = _mm256_broadcast_sd(&gate[0].imag);
4236  const __m256d u01r = _mm256_broadcast_sd(&gate[1].real);
4237  const __m256d u01i = _mm256_broadcast_sd(&gate[1].imag);
4238  const __m256d u10r = _mm256_broadcast_sd(&gate[2].real);
4239  const __m256d u10i = _mm256_broadcast_sd(&gate[2].imag);
4240  const __m256d u11r = _mm256_broadcast_sd(&gate[3].real);
4241  const __m256d u11i = _mm256_broadcast_sd(&gate[3].imag);
4242  double* element = (double*)input.get_data() + 2 * (row_offset + current_idx);
4243  double* element_pair = (double*)input.get_data() + 2 * (row_offset + current_idx_pair);
4244  int col_idx = 0;
4245  const int avx_limit = 2 * (index_step_target - 3);
4246 
4247  for (; col_idx < avx_limit; col_idx += 8) {
4248  __m256d element_vec = _mm256_loadu_pd(element + col_idx);
4249  __m256d element_vec2 = _mm256_loadu_pd(element + col_idx + 4);
4250  __m256d tmp = _mm256_shuffle_pd(element_vec, element_vec2, 0);
4251  element_vec2 = _mm256_shuffle_pd(element_vec, element_vec2, 0xf);
4252  element_vec = tmp;
4253  __m256d element_pair_vec = _mm256_loadu_pd(element_pair + col_idx);
4254  __m256d element_pair_vec2 = _mm256_loadu_pd(element_pair + col_idx + 4);
4255  tmp = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0);
4256  element_pair_vec2 = _mm256_shuffle_pd(element_pair_vec, element_pair_vec2, 0xf);
4257  element_pair_vec = tmp;
4258 
4259  __m256d vec3 = _mm256_mul_pd(u00r, element_vec);
4260  vec3 = _mm256_fnmadd_pd(u00i, element_vec2, vec3);
4261  __m256d vec4 = _mm256_mul_pd(u10r, element_pair_vec);
4262  vec4 = _mm256_fnmadd_pd(u10i, element_pair_vec2, vec4);
4263  vec3 = _mm256_add_pd(vec3, vec4);
4264  __m256d vec5 = _mm256_mul_pd(u00r, element_vec2);
4265  vec5 = _mm256_fmadd_pd(u00i, element_vec, vec5);
4266  __m256d vec6 = _mm256_mul_pd(u10r, element_pair_vec2);
4267  vec6 = _mm256_fmadd_pd(u10i, element_pair_vec, vec6);
4268  vec5 = _mm256_add_pd(vec5, vec6);
4269  tmp = _mm256_shuffle_pd(vec3, vec5, 0);
4270  vec5 = _mm256_shuffle_pd(vec3, vec5, 0xf);
4271  vec3 = tmp;
4272  _mm256_storeu_pd(element + col_idx, vec3);
4273  _mm256_storeu_pd(element + col_idx + 4, vec5);
4274 
4275  __m256d vec7 = _mm256_mul_pd(u01r, element_vec);
4276  vec7 = _mm256_fnmadd_pd(u01i, element_vec2, vec7);
4277  __m256d vec8 = _mm256_mul_pd(u11r, element_pair_vec);
4278  vec8 = _mm256_fnmadd_pd(u11i, element_pair_vec2, vec8);
4279  vec7 = _mm256_add_pd(vec7, vec8);
4280  __m256d vec9 = _mm256_mul_pd(u01r, element_vec2);
4281  vec9 = _mm256_fmadd_pd(u01i, element_vec, vec9);
4282  __m256d vec10 = _mm256_mul_pd(u11r, element_pair_vec2);
4283  vec10 = _mm256_fmadd_pd(u11i, element_pair_vec, vec10);
4284  vec9 = _mm256_add_pd(vec9, vec10);
4285  tmp = _mm256_shuffle_pd(vec7, vec9, 0);
4286  vec9 = _mm256_shuffle_pd(vec7, vec9, 0xf);
4287  vec7 = tmp;
4288  _mm256_storeu_pd(element_pair + col_idx, vec7);
4289  _mm256_storeu_pd(element_pair + col_idx + 4, vec9);
4290  }
4291 
4292  for (int c = col_idx / 2; c < index_step_target; ++c) {
4293  const int index = row_offset + current_idx + c;
4294  const int index_pair = row_offset + current_idx_pair + c;
4295  QGD_Complex16 a = input[index];
4296  QGD_Complex16 b = input[index_pair];
4297  QGD_Complex16 tmp1 = mult(gate[0], a);
4298  QGD_Complex16 tmp2 = mult(gate[2], b);
4299  input[index].real = tmp1.real + tmp2.real;
4300  input[index].imag = tmp1.imag + tmp2.imag;
4301  tmp1 = mult(gate[1], a);
4302  tmp2 = mult(gate[3], b);
4303  input[index_pair].real = tmp1.real + tmp2.real;
4304  input[index_pair].imag = tmp1.imag + tmp2.imag;
4305  }
4306  } else {
4307  for (int idx = 0; idx < index_step_target; ++idx) {
4308  const int current_idx_loc = current_idx + idx;
4309  const int current_idx_pair_loc = current_idx_pair + idx;
4310  Matrix& gate = ((current_idx_loc >> control_qbit) & 1) ? u3_1qbit1 : u3_1qbit2;
4311  const int index = row_offset + current_idx_loc;
4312  const int index_pair = row_offset + current_idx_pair_loc;
4313  QGD_Complex16 a = input[index];
4314  QGD_Complex16 b = input[index_pair];
4315  QGD_Complex16 tmp1 = mult(gate[0], a);
4316  QGD_Complex16 tmp2 = mult(gate[2], b);
4317  input[index].real = tmp1.real + tmp2.real;
4318  input[index].imag = tmp1.imag + tmp2.imag;
4319  tmp1 = mult(gate[1], a);
4320  tmp2 = mult(gate[3], b);
4321  input[index_pair].real = tmp1.real + tmp2.real;
4322  input[index_pair].imag = tmp1.imag + tmp2.imag;
4323  }
4324  }
4325 
4326  current_idx += (index_step_target << 1);
4327  current_idx_pair += (index_step_target << 1);
4328  }
4329  }
4330  });
4331 
4332  (void)matrix_size;
4333 }
4334 
4335 template<int n>
4336 void apply_fixed_qbit_unitary_AVX32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size);
4337 
4338 template<int n>
4339 void apply_fixed_qbit_unitary_AVX_OpenMP32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size);
4340 
4341 template<int n>
4342 void apply_fixed_qbit_unitary_AVX_TBB32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size);
4343 
4344 
4345 void apply_large_kernel_to_input_AVX32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
4346  if (input.cols == 1) {
4347  switch (involved_qbits.size()) {
4348  case 2:
4349  apply_2qbit_kernel_to_state_vector_input_AVX32(unitary, input, involved_qbits, matrix_size);
4350  break;
4351  case 3:
4352  apply_3qbit_kernel_to_state_vector_input_AVX32(unitary, input, involved_qbits, matrix_size);
4353  break;
4354  case 4:
4355  apply_4qbit_kernel_to_state_vector_input_AVX32(unitary, input, involved_qbits, matrix_size);
4356  break;
4357  case 5:
4358  apply_5qbit_kernel_to_state_vector_input_AVX32(unitary, input, involved_qbits, matrix_size);
4359  break;
4360  default:
4361  break;
4362  }
4363  } else {
4364  switch (involved_qbits.size()) {
4365  case 2: {
4366  apply_fixed_qbit_unitary_AVX32<2>(unitary, input, involved_qbits, matrix_size);
4367  break;
4368  }
4369  case 3: {
4370  apply_fixed_qbit_unitary_AVX32<3>(unitary, input, involved_qbits, matrix_size);
4371  break;
4372  }
4373  case 4: {
4374  apply_fixed_qbit_unitary_AVX32<4>(unitary, input, involved_qbits, matrix_size);
4375  break;
4376  }
4377  case 5: {
4378  apply_fixed_qbit_unitary_AVX32<5>(unitary, input, involved_qbits, matrix_size);
4379  break;
4380  }
4381  default: {
4382  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
4383  break;
4384  }
4385  }
4386  }
4387 }
4388 
4389 void apply_large_kernel_to_input_AVX_OpenMP32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
4390  if (input.cols == 1) {
4391  switch (involved_qbits.size()) {
4392  case 2:
4393  apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32(unitary, input, involved_qbits, matrix_size);
4394  break;
4395  case 3:
4396  apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32(unitary, input, involved_qbits, matrix_size);
4397  break;
4398  case 4:
4399  apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32(unitary, input, involved_qbits, matrix_size);
4400  break;
4401  case 5:
4402  apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32(unitary, input, involved_qbits, matrix_size);
4403  break;
4404  default:
4405  break;
4406  }
4407  } else {
4408  switch (involved_qbits.size()) {
4409  case 2: {
4410  apply_fixed_qbit_unitary_AVX_OpenMP32<2>(unitary, input, involved_qbits, matrix_size);
4411  break;
4412  }
4413  case 3: {
4414  apply_fixed_qbit_unitary_AVX_OpenMP32<3>(unitary, input, involved_qbits, matrix_size);
4415  break;
4416  }
4417  case 4: {
4418  apply_fixed_qbit_unitary_AVX_OpenMP32<4>(unitary, input, involved_qbits, matrix_size);
4419  break;
4420  }
4421  case 5: {
4422  apply_fixed_qbit_unitary_AVX_OpenMP32<5>(unitary, input, involved_qbits, matrix_size);
4423  break;
4424  }
4425  default: {
4426  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
4427  break;
4428  }
4429  }
4430  }
4431 }
4432 
4433 void apply_large_kernel_to_input_AVX_TBB32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
4434  if (input.cols == 1) {
4435  switch (involved_qbits.size()) {
4436  case 2:
4437  apply_2qbit_kernel_to_state_vector_input_AVX_TBB32(unitary, input, involved_qbits, matrix_size);
4438  break;
4439  case 3:
4440  apply_3qbit_kernel_to_state_vector_input_AVX_TBB32(unitary, input, involved_qbits, matrix_size);
4441  break;
4442  case 4:
4443  apply_4qbit_kernel_to_state_vector_input_AVX_TBB32(unitary, input, involved_qbits, matrix_size);
4444  break;
4445  case 5:
4446  apply_5qbit_kernel_to_state_vector_input_AVX_TBB32(unitary, input, involved_qbits, matrix_size);
4447  break;
4448  default:
4449  break;
4450  }
4451  } else {
4452  switch (involved_qbits.size()) {
4453  case 2: {
4454  apply_fixed_qbit_unitary_AVX_TBB32<2>(unitary, input, involved_qbits, matrix_size);
4455  break;
4456  }
4457  case 3: {
4458  apply_fixed_qbit_unitary_AVX_TBB32<3>(unitary, input, involved_qbits, matrix_size);
4459  break;
4460  }
4461  case 4: {
4462  apply_fixed_qbit_unitary_AVX_TBB32<4>(unitary, input, involved_qbits, matrix_size);
4463  break;
4464  }
4465  case 5: {
4466  apply_fixed_qbit_unitary_AVX_TBB32<5>(unitary, input, involved_qbits, matrix_size);
4467  break;
4468  }
4469  default: {
4470  apply_large_kernel_to_input(unitary, input, std::move(involved_qbits), matrix_size);
4471  break;
4472  }
4473  }
4474  }
4475 }
4476 
4477 template<int n>
4478 inline void construct_mv_xy_vectors32_fixed(const Matrix_float& gate_kernel_unitary, __m256* mv_xy);
4479 
4480 inline __m256* construct_mv_xy_vectors32(const Matrix_float& gate_kernel_unitary, const int& matrix_size);
4481 
4482 template<int n>
4483 inline void construct_mv_xy_vectors_fixed(const Matrix& gate_kernel_unitary, __m256d* mv_xy) {
4484  constexpr int block_size = 1 << n;
4485 
4486  for (int rdx = 0; rdx < block_size; ++rdx) {
4487  for (int cdx = 0; cdx < block_size; cdx += 2) {
4488  mv_xy[rdx * block_size + cdx] = _mm256_set_pd(
4489  -gate_kernel_unitary[block_size * rdx + cdx + 1].imag,
4490  gate_kernel_unitary[block_size * rdx + cdx + 1].real,
4491  -gate_kernel_unitary[block_size * rdx + cdx].imag,
4492  gate_kernel_unitary[block_size * rdx + cdx].real
4493  );
4494 
4495  mv_xy[rdx * block_size + cdx + 1] = _mm256_set_pd(
4496  gate_kernel_unitary[block_size * rdx + cdx + 1].real,
4497  gate_kernel_unitary[block_size * rdx + cdx + 1].imag,
4498  gate_kernel_unitary[block_size * rdx + cdx].real,
4499  gate_kernel_unitary[block_size * rdx + cdx].imag
4500  );
4501  }
4502  }
4503 }
4504 
4505 template<int n>
4506 inline void complex_prod_AVX_fixed(const __m256d* mv_xy, int rdx, int cdx, const int* indices, const Matrix& input, int col, __m256d& result) {
4507  constexpr int block_size = 1 << n;
4508  const double* data_ptr = (const double*)input.get_data();
4509  const int stride = input.stride;
4510 
4511  const int idx0 = indices[cdx] * stride + col;
4512  const int idx1 = indices[cdx + 1] * stride + col;
4513 
4514  const __m256d data = _mm256_set_pd(
4515  data_ptr[2 * idx1 + 1],
4516  data_ptr[2 * idx1 + 0],
4517  data_ptr[2 * idx0 + 1],
4518  data_ptr[2 * idx0 + 0]
4519  );
4520 
4521  const __m256d mv_x0 = mv_xy[block_size * rdx + cdx];
4522  const __m256d mv_x1 = mv_xy[block_size * rdx + cdx + 1];
4523  result = _mm256_add_pd(result, _mm256_hadd_pd(_mm256_mul_pd(data, mv_x0), _mm256_mul_pd(data, mv_x1)));
4524 }
4525 
4526 template<int n>
4527 void apply_fixed_qbit_unitary_AVX(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4528  constexpr int block_size = 1 << n;
4529  const int qubit_num = (int) std::log2(matrix_size);
4530  const int num_blocks = matrix_size >> n;
4531  std::sort(involved_qbits.begin(), involved_qbits.end());
4532 
4533  int non_targets[64];
4534  int non_target_count = 0;
4535  for (int q = 0; q < qubit_num; ++q) {
4536  bool is_target = false;
4537  for (int target : involved_qbits) {
4538  is_target = is_target || (q == target);
4539  }
4540  if (!is_target) {
4541  non_targets[non_target_count++] = q;
4542  }
4543  }
4544 
4545  int block_pattern[block_size];
4546  for (int k = 0; k < block_size; ++k) {
4547  int idx = 0;
4548  for (int bit = 0; bit < n; ++bit) {
4549  if (k & (1 << bit)) {
4550  idx |= (1 << involved_qbits[bit]);
4551  }
4552  }
4553  block_pattern[k] = idx;
4554  }
4555 
4556  int indices[block_size];
4557  QGD_Complex16 out[block_size];
4558  __m256d mv_xy[block_size * block_size];
4559  construct_mv_xy_vectors_fixed<n>(gate_kernel_unitary, mv_xy);
4560 
4561  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
4562  int base = 0;
4563  for (int i = 0; i < non_target_count; ++i) {
4564  if (iter_idx & (1ULL << i)) {
4565  base |= (1 << non_targets[i]);
4566  }
4567  }
4568  for (int k = 0; k < block_size; ++k) {
4569  indices[k] = base | block_pattern[k];
4570  }
4571 
4572  for (int col = 0; col < input.cols; ++col) {
4573  for (int rdx = 0; rdx < block_size; ++rdx) {
4574  __m256d result = _mm256_setzero_pd();
4575 
4576  for (int cdx = 0; cdx < block_size; cdx += 2) {
4577  complex_prod_AVX_fixed<n>(mv_xy, rdx, cdx, indices, input, col, result);
4578  }
4579 
4580  __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4581  __m256d sum = _mm256_add_pd(result, perm);
4582  __m128d low128 = _mm256_castpd256_pd128(sum);
4583  out[rdx].real = _mm_cvtsd_f64(low128);
4584  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4585  }
4586 
4587  for (int rdx = 0; rdx < block_size; ++rdx) {
4588  input[indices[rdx] * input.stride + col] = out[rdx];
4589  }
4590  }
4591  }
4592 }
4593 
4594 template<int n>
4595 void apply_fixed_qbit_unitary_AVX_OpenMP(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4596  constexpr int block_size = 1 << n;
4597  const int qubit_num = (int) std::log2(matrix_size);
4598  const int num_blocks = matrix_size >> n;
4599  std::sort(involved_qbits.begin(), involved_qbits.end());
4600 
4601  int non_targets[64];
4602  int non_target_count = 0;
4603  for (int q = 0; q < qubit_num; ++q) {
4604  bool is_target = false;
4605  for (int target : involved_qbits) {
4606  is_target = is_target || (q == target);
4607  }
4608  if (!is_target) {
4609  non_targets[non_target_count++] = q;
4610  }
4611  }
4612 
4613  int block_pattern[block_size];
4614  for (int k = 0; k < block_size; ++k) {
4615  int idx = 0;
4616  for (int bit = 0; bit < n; ++bit) {
4617  if (k & (1 << bit)) {
4618  idx |= (1 << involved_qbits[bit]);
4619  }
4620  }
4621  block_pattern[k] = idx;
4622  }
4623 
4624  __m256d mv_xy[block_size * block_size];
4625  construct_mv_xy_vectors_fixed<n>(gate_kernel_unitary, mv_xy);
4626 
4627 #pragma omp parallel for schedule(static)
4628  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
4629  int indices[block_size];
4630  QGD_Complex16 out[block_size];
4631 
4632  int base = 0;
4633  for (int i = 0; i < non_target_count; ++i) {
4634  if (iter_idx & (1ULL << i)) {
4635  base |= (1 << non_targets[i]);
4636  }
4637  }
4638  for (int k = 0; k < block_size; ++k) {
4639  indices[k] = base | block_pattern[k];
4640  }
4641 
4642  for (int col = 0; col < input.cols; ++col) {
4643  for (int rdx = 0; rdx < block_size; ++rdx) {
4644  __m256d result = _mm256_setzero_pd();
4645 
4646  for (int cdx = 0; cdx < block_size; cdx += 2) {
4647  complex_prod_AVX_fixed<n>(mv_xy, rdx, cdx, indices, input, col, result);
4648  }
4649 
4650  __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4651  __m256d sum = _mm256_add_pd(result, perm);
4652  __m128d low128 = _mm256_castpd256_pd128(sum);
4653  out[rdx].real = _mm_cvtsd_f64(low128);
4654  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4655  }
4656 
4657  for (int rdx = 0; rdx < block_size; ++rdx) {
4658  input[indices[rdx] * input.stride + col] = out[rdx];
4659  }
4660  }
4661  }
4662 }
4663 
4664 template<int n>
4665 void apply_fixed_qbit_unitary_AVX_TBB(Matrix& gate_kernel_unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4666  constexpr int block_size = 1 << n;
4667  const int qubit_num = (int) std::log2(matrix_size);
4668  const int num_blocks = matrix_size >> n;
4669  std::sort(involved_qbits.begin(), involved_qbits.end());
4670 
4671  int non_targets[64];
4672  int non_target_count = 0;
4673  for (int q = 0; q < qubit_num; ++q) {
4674  bool is_target = false;
4675  for (int target : involved_qbits) {
4676  is_target = is_target || (q == target);
4677  }
4678  if (!is_target) {
4679  non_targets[non_target_count++] = q;
4680  }
4681  }
4682 
4683  int block_pattern[block_size];
4684  for (int k = 0; k < block_size; ++k) {
4685  int idx = 0;
4686  for (int bit = 0; bit < n; ++bit) {
4687  if (k & (1 << bit)) {
4688  idx |= (1 << involved_qbits[bit]);
4689  }
4690  }
4691  block_pattern[k] = idx;
4692  }
4693 
4694  __m256d mv_xy[block_size * block_size];
4695  construct_mv_xy_vectors_fixed<n>(gate_kernel_unitary, mv_xy);
4696 
4697  tbb::parallel_for(tbb::blocked_range<int>(0, num_blocks, 16), [&](const tbb::blocked_range<int>& range) {
4698  for (int iter_idx = range.begin(); iter_idx < range.end(); ++iter_idx) {
4699  std::array<int, 1 << n> indices;
4700  std::array<QGD_Complex16, 1 << n> out;
4701 
4702  int base = 0;
4703  for (int i = 0; i < non_target_count; ++i) {
4704  if (iter_idx & (1ULL << i)) {
4705  base |= (1 << non_targets[i]);
4706  }
4707  }
4708  for (int k = 0; k < block_size; ++k) {
4709  indices[k] = base | block_pattern[k];
4710  }
4711 
4712  for (int col = 0; col < input.cols; ++col) {
4713  for (int rdx = 0; rdx < block_size; ++rdx) {
4714  __m256d result = _mm256_setzero_pd();
4715 
4716  for (int cdx = 0; cdx < block_size; cdx += 2) {
4717  complex_prod_AVX_fixed<n>(mv_xy, rdx, cdx, indices.data(), input, col, result);
4718  }
4719 
4720  __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4721  __m256d sum = _mm256_add_pd(result, perm);
4722  __m128d low128 = _mm256_castpd256_pd128(sum);
4723  out[rdx].real = _mm_cvtsd_f64(low128);
4724  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4725  }
4726 
4727  for (int rdx = 0; rdx < block_size; ++rdx) {
4728  input[indices[rdx] * input.stride + col] = out[rdx];
4729  }
4730  }
4731  }
4732  });
4733 }
4734 
4735 namespace {
4736 
4737 template<typename MatrixT>
4738 MatrixT transpose_local_kernel(const MatrixT& unitary) {
4739  MatrixT transposed = unitary.copy();
4740  for (int row = 0; row < unitary.rows; ++row) {
4741  for (int col = 0; col < unitary.cols; ++col) {
4742  transposed[row * unitary.cols + col] = unitary[col * unitary.cols + row];
4743  }
4744  }
4745  return transposed;
4746 }
4747 
4748 inline Matrix transpose_local_kernel_4x4(const Matrix& unitary) {
4749  Matrix transposed = unitary.copy();
4750  for (int row = 0; row < 4; ++row) {
4751  for (int col = 0; col < 4; ++col) {
4752  transposed[row * 4 + col] = unitary[col * 4 + row];
4753  }
4754  }
4755  return transposed;
4756 }
4757 
4758 inline Matrix_float transpose_local_kernel_4x4(const Matrix_float& unitary) {
4759  Matrix_float transposed = unitary.copy();
4760  for (int row = 0; row < 4; ++row) {
4761  for (int col = 0; col < 4; ++col) {
4762  transposed[row * 4 + col] = unitary[col * 4 + row];
4763  }
4764  }
4765  return transposed;
4766 }
4767 
4768 inline void complex_prod_AVX_from_right_generic(const __m256d* mv_xy, int block_size, int rdx, int cdx, const int* indices, const Matrix& input, int row_offset, __m256d& result) {
4769  const int idx0 = row_offset + indices[cdx];
4770  const int idx1 = row_offset + indices[cdx + 1];
4771  const double* data_ptr = (const double*)input.get_data();
4772 
4773  const __m256d data = _mm256_set_pd(
4774  data_ptr[2 * idx1 + 1],
4775  data_ptr[2 * idx1 + 0],
4776  data_ptr[2 * idx0 + 1],
4777  data_ptr[2 * idx0 + 0]
4778  );
4779 
4780  const __m256d mv_x0 = mv_xy[block_size * rdx + cdx];
4781  const __m256d mv_x1 = mv_xy[block_size * rdx + cdx + 1];
4782  const __m256d data_u0 = _mm256_mul_pd(data, mv_x0);
4783  const __m256d data_u1 = _mm256_mul_pd(data, mv_x1);
4784  result = _mm256_add_pd(result, _mm256_hadd_pd(data_u0, data_u1));
4785 }
4786 
4787 inline void complex_prod_AVX_from_right(const __m256d* mv_xy, int rdx, int cdx, const int* indices, const Matrix& input, int row_offset, __m256d& result) {
4788  const int block_size = 4;
4789  const int idx0 = row_offset + indices[cdx];
4790  const int idx1 = row_offset + indices[cdx + 1];
4791  const double* data_ptr = (const double*)input.get_data();
4792 
4793  const __m256d data = _mm256_set_pd(
4794  data_ptr[2 * idx1 + 1],
4795  data_ptr[2 * idx1 + 0],
4796  data_ptr[2 * idx0 + 1],
4797  data_ptr[2 * idx0 + 0]
4798  );
4799 
4800  const __m256d mv_x0 = mv_xy[block_size * rdx + cdx];
4801  const __m256d mv_x1 = mv_xy[block_size * rdx + cdx + 1];
4802  const __m256d data_u0 = _mm256_mul_pd(data, mv_x0);
4803  const __m256d data_u1 = _mm256_mul_pd(data, mv_x1);
4804  result = _mm256_add_pd(result, _mm256_hadd_pd(data_u0, data_u1));
4805 }
4806 
4807 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4808  if (input.cols == 1) {
4809  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
4810  }
4811 
4812  const int n = static_cast<int>(involved_qbits.size());
4813  const int block_size = 1 << n;
4814  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
4815  throw std::invalid_argument("AVX right-apply large-kernel path received an unsupported local kernel size.");
4816  }
4817 
4818  const int qubit_num = (int)std::log2(matrix_size);
4819  std::sort(involved_qbits.begin(), involved_qbits.end());
4820 
4821  std::vector<int> non_targets;
4822  non_targets.reserve(qubit_num - n);
4823  for (int q = 0; q < qubit_num; ++q) {
4824  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
4825  non_targets.push_back(q);
4826  }
4827  }
4828 
4829  std::vector<int> block_pattern(block_size);
4830  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
4831  std::vector<int> indices(block_size);
4832  std::vector<QGD_Complex16> out(block_size);
4833  const int num_blocks = matrix_size >> n;
4834  const Matrix transposed = transpose_local_kernel(unitary);
4835  __m256d* mv_xy = construct_mv_xy_vectors(transposed, block_size);
4836 
4837  for (int row = 0; row < input.rows; ++row) {
4838  const int row_offset = row * input.stride;
4839 
4840  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
4841  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
4842 
4843  for (int rdx = 0; rdx < block_size; ++rdx) {
4844  __m256d result = _mm256_setzero_pd();
4845 
4846  for (int cdx = 0; cdx < block_size; cdx += 2) {
4847  complex_prod_AVX_from_right_generic(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
4848  }
4849 
4850  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4851  const __m256d sum = _mm256_add_pd(result, perm);
4852  const __m128d low128 = _mm256_castpd256_pd128(sum);
4853  out[rdx].real = _mm_cvtsd_f64(low128);
4854  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4855  }
4856 
4857  for (int rdx = 0; rdx < block_size; ++rdx) {
4858  input[row_offset + indices[rdx]] = out[rdx];
4859  }
4860  }
4861  }
4862 
4863  _mm_free(mv_xy);
4864 }
4865 
4866 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX_OpenMP_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4867  if (input.cols == 1) {
4868  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
4869  }
4870 
4871  const int n = static_cast<int>(involved_qbits.size());
4872  const int block_size = 1 << n;
4873  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
4874  throw std::invalid_argument("AVX right-apply large-kernel path received an unsupported local kernel size.");
4875  }
4876 
4877  const int qubit_num = (int)std::log2(matrix_size);
4878  std::sort(involved_qbits.begin(), involved_qbits.end());
4879 
4880  std::vector<int> non_targets;
4881  non_targets.reserve(qubit_num - n);
4882  for (int q = 0; q < qubit_num; ++q) {
4883  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
4884  non_targets.push_back(q);
4885  }
4886  }
4887 
4888  std::vector<int> block_pattern(block_size);
4889  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
4890  const int num_blocks = matrix_size >> n;
4891  const Matrix transposed = transpose_local_kernel(unitary);
4892  __m256d* mv_xy = construct_mv_xy_vectors(transposed, block_size);
4893 
4894 #pragma omp parallel for schedule(static)
4895  for (int row = 0; row < input.rows; ++row) {
4896  const int row_offset = row * input.stride;
4897  std::vector<int> indices(block_size);
4898  std::vector<QGD_Complex16> out(block_size);
4899 
4900  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
4901  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
4902 
4903  for (int rdx = 0; rdx < block_size; ++rdx) {
4904  __m256d result = _mm256_setzero_pd();
4905 
4906  for (int cdx = 0; cdx < block_size; cdx += 2) {
4907  complex_prod_AVX_from_right_generic(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
4908  }
4909 
4910  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4911  const __m256d sum = _mm256_add_pd(result, perm);
4912  const __m128d low128 = _mm256_castpd256_pd128(sum);
4913  out[rdx].real = _mm_cvtsd_f64(low128);
4914  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4915  }
4916 
4917  for (int rdx = 0; rdx < block_size; ++rdx) {
4918  input[row_offset + indices[rdx]] = out[rdx];
4919  }
4920  }
4921  }
4922 
4923  _mm_free(mv_xy);
4924 }
4925 
4926 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX_TBB_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
4927  if (input.cols == 1) {
4928  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
4929  }
4930 
4931  const int n = static_cast<int>(involved_qbits.size());
4932  const int block_size = 1 << n;
4933  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
4934  throw std::invalid_argument("AVX right-apply large-kernel path received an unsupported local kernel size.");
4935  }
4936 
4937  const int qubit_num = (int)std::log2(matrix_size);
4938  std::sort(involved_qbits.begin(), involved_qbits.end());
4939 
4940  std::vector<int> non_targets;
4941  non_targets.reserve(qubit_num - n);
4942  for (int q = 0; q < qubit_num; ++q) {
4943  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
4944  non_targets.push_back(q);
4945  }
4946  }
4947 
4948  std::vector<int> block_pattern(block_size);
4949  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
4950  const int num_blocks = matrix_size >> n;
4951  const Matrix transposed = transpose_local_kernel(unitary);
4952  __m256d* mv_xy = construct_mv_xy_vectors(transposed, block_size);
4953 
4954  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
4955  for (int row = range.begin(); row < range.end(); ++row) {
4956  const int row_offset = row * input.stride;
4957  std::vector<int> indices(block_size);
4958  std::vector<QGD_Complex16> out(block_size);
4959 
4960  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
4961  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
4962 
4963  for (int rdx = 0; rdx < block_size; ++rdx) {
4964  __m256d result = _mm256_setzero_pd();
4965 
4966  for (int cdx = 0; cdx < block_size; cdx += 2) {
4967  complex_prod_AVX_from_right_generic(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
4968  }
4969 
4970  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
4971  const __m256d sum = _mm256_add_pd(result, perm);
4972  const __m128d low128 = _mm256_castpd256_pd128(sum);
4973  out[rdx].real = _mm_cvtsd_f64(low128);
4974  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
4975  }
4976 
4977  for (int rdx = 0; rdx < block_size; ++rdx) {
4978  input[row_offset + indices[rdx]] = out[rdx];
4979  }
4980  }
4981  }
4982  });
4983 
4984  _mm_free(mv_xy);
4985 }
4986 
4987 inline void build_2qbit_block_pattern(std::vector<int>& involved_qbits, int* non_targets, int& non_target_count, int* block_pattern, int qubit_num) {
4988  std::sort(involved_qbits.begin(), involved_qbits.end());
4989 
4990  non_target_count = 0;
4991  for (int q = 0; q < qubit_num; ++q) {
4992  if (q != involved_qbits[0] && q != involved_qbits[1]) {
4993  non_targets[non_target_count++] = q;
4994  }
4995  }
4996 
4997  for (int k = 0; k < 4; ++k) {
4998  int idx = 0;
4999  for (int bit = 0; bit < 2; ++bit) {
5000  if (k & (1 << bit)) {
5001  idx |= (1 << involved_qbits[bit]);
5002  }
5003  }
5004  block_pattern[k] = idx;
5005  }
5006 }
5007 
5008 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5009  if (input.cols == 1) {
5010  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5011  }
5012  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5013  throw std::invalid_argument("AVX right-apply large-kernel path currently supports 4x4 local kernels only.");
5014  }
5015 
5016  const int qubit_num = (int)std::log2(matrix_size);
5017  int non_targets[64];
5018  int non_target_count = 0;
5019  int block_pattern[4];
5020  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5021 
5022  const Matrix transposed = transpose_local_kernel_4x4(unitary);
5023  __m256d* mv_xy = construct_mv_xy_vectors(transposed, 4);
5024  int indices[4];
5025  QGD_Complex16 out[4];
5026  const int num_blocks = matrix_size >> 2;
5027 
5028  for (int row = 0; row < input.rows; ++row) {
5029  const int row_offset = row * input.stride;
5030 
5031  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5032  int base = 0;
5033  for (int i = 0; i < non_target_count; ++i) {
5034  if (iter_idx & (1ULL << i)) {
5035  base |= (1 << non_targets[i]);
5036  }
5037  }
5038  for (int k = 0; k < 4; ++k) {
5039  indices[k] = base | block_pattern[k];
5040  }
5041 
5042  for (int rdx = 0; rdx < 4; ++rdx) {
5043  __m256d result = _mm256_setzero_pd();
5044 
5045  for (int cdx = 0; cdx < 4; cdx += 2) {
5046  complex_prod_AVX_from_right(mv_xy, rdx, cdx, indices, input, row_offset, result);
5047  }
5048 
5049  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
5050  const __m256d sum = _mm256_add_pd(result, perm);
5051  const __m128d low128 = _mm256_castpd256_pd128(sum);
5052  out[rdx].real = _mm_cvtsd_f64(low128);
5053  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
5054  }
5055 
5056  for (int rdx = 0; rdx < 4; ++rdx) {
5057  input[row_offset + indices[rdx]] = out[rdx];
5058  }
5059  }
5060  }
5061 
5062  _mm_free(mv_xy);
5063 }
5064 
5065 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX_OpenMP_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5066  if (input.cols == 1) {
5067  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5068  }
5069  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5070  throw std::invalid_argument("AVX right-apply large-kernel path currently supports 4x4 local kernels only.");
5071  }
5072 
5073  const int qubit_num = (int)std::log2(matrix_size);
5074  int non_targets[64];
5075  int non_target_count = 0;
5076  int block_pattern[4];
5077  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5078 
5079  const Matrix transposed = transpose_local_kernel_4x4(unitary);
5080  __m256d* mv_xy = construct_mv_xy_vectors(transposed, 4);
5081  const int num_blocks = matrix_size >> 2;
5082 
5083 #pragma omp parallel for schedule(static)
5084  for (int row = 0; row < input.rows; ++row) {
5085  const int row_offset = row * input.stride;
5086  int indices[4];
5087  QGD_Complex16 out[4];
5088 
5089  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5090  int base = 0;
5091  for (int i = 0; i < non_target_count; ++i) {
5092  if (iter_idx & (1ULL << i)) {
5093  base |= (1 << non_targets[i]);
5094  }
5095  }
5096  for (int k = 0; k < 4; ++k) {
5097  indices[k] = base | block_pattern[k];
5098  }
5099 
5100  for (int rdx = 0; rdx < 4; ++rdx) {
5101  __m256d result = _mm256_setzero_pd();
5102  for (int cdx = 0; cdx < 4; cdx += 2) {
5103  complex_prod_AVX_from_right(mv_xy, rdx, cdx, indices, input, row_offset, result);
5104  }
5105 
5106  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
5107  const __m256d sum = _mm256_add_pd(result, perm);
5108  const __m128d low128 = _mm256_castpd256_pd128(sum);
5109  out[rdx].real = _mm_cvtsd_f64(low128);
5110  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
5111  }
5112 
5113  for (int rdx = 0; rdx < 4; ++rdx) {
5114  input[row_offset + indices[rdx]] = out[rdx];
5115  }
5116  }
5117  }
5118 
5119  _mm_free(mv_xy);
5120 }
5121 
5122 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX_TBB_impl(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5123  if (input.cols == 1) {
5124  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5125  }
5126  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5127  throw std::invalid_argument("AVX right-apply large-kernel path currently supports 4x4 local kernels only.");
5128  }
5129 
5130  const int qubit_num = (int)std::log2(matrix_size);
5131  int non_targets[64];
5132  int non_target_count = 0;
5133  int block_pattern[4];
5134  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5135 
5136  const Matrix transposed = transpose_local_kernel_4x4(unitary);
5137  __m256d* mv_xy = construct_mv_xy_vectors(transposed, 4);
5138  const int num_blocks = matrix_size >> 2;
5139 
5140  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
5141  for (int row = range.begin(); row < range.end(); ++row) {
5142  const int row_offset = row * input.stride;
5143  int indices[4];
5144  QGD_Complex16 out[4];
5145 
5146  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5147  int base = 0;
5148  for (int i = 0; i < non_target_count; ++i) {
5149  if (iter_idx & (1ULL << i)) {
5150  base |= (1 << non_targets[i]);
5151  }
5152  }
5153  for (int k = 0; k < 4; ++k) {
5154  indices[k] = base | block_pattern[k];
5155  }
5156 
5157  for (int rdx = 0; rdx < 4; ++rdx) {
5158  __m256d result = _mm256_setzero_pd();
5159  for (int cdx = 0; cdx < 4; cdx += 2) {
5160  complex_prod_AVX_from_right(mv_xy, rdx, cdx, indices, input, row_offset, result);
5161  }
5162 
5163  const __m256d perm = _mm256_permute2f128_pd(result, result, 0x01);
5164  const __m256d sum = _mm256_add_pd(result, perm);
5165  const __m128d low128 = _mm256_castpd256_pd128(sum);
5166  out[rdx].real = _mm_cvtsd_f64(low128);
5167  out[rdx].imag = _mm_cvtsd_f64(_mm_unpackhi_pd(low128, low128));
5168  }
5169 
5170  for (int rdx = 0; rdx < 4; ++rdx) {
5171  input[row_offset + indices[rdx]] = out[rdx];
5172  }
5173  }
5174  }
5175  });
5176 
5177  _mm_free(mv_xy);
5178 }
5179 
5180 template<int n>
5181 inline void complex_prod_AVX32_fixed_from_right(const __m256* mv_xy, int rdx, int cdx, const int* indices, const Matrix_float& input, int row_offset, __m256& result) {
5182  constexpr int block_size = 1 << n;
5183  const float* data_ptr = (const float*)input.get_data();
5184 
5185  const int idx0 = row_offset + indices[cdx];
5186  const int idx1 = row_offset + indices[cdx + 1];
5187  const int idx2 = row_offset + indices[cdx + 2];
5188  const int idx3 = row_offset + indices[cdx + 3];
5189 
5190  const __m256 data = _mm256_set_ps(
5191  data_ptr[2 * idx3 + 1],
5192  data_ptr[2 * idx3 + 0],
5193  data_ptr[2 * idx2 + 1],
5194  data_ptr[2 * idx2 + 0],
5195  data_ptr[2 * idx1 + 1],
5196  data_ptr[2 * idx1 + 0],
5197  data_ptr[2 * idx0 + 1],
5198  data_ptr[2 * idx0 + 0]
5199  );
5200 
5201  const __m256 mv_x0 = mv_xy[block_size * rdx + cdx];
5202  const __m256 mv_x1 = mv_xy[block_size * rdx + cdx + 1];
5203  result = _mm256_add_ps(result, _mm256_hadd_ps(_mm256_mul_ps(data, mv_x0), _mm256_mul_ps(data, mv_x1)));
5204 }
5205 
5206 inline void complex_prod_AVX32_from_right(const __m256* mv_xy, int block_size, int rdx, int cdx, const int* indices, const Matrix_float& input, int row_offset, __m256& result) {
5207  const float* data_ptr = (const float*)input.get_data();
5208 
5209  const int idx0 = row_offset + indices[cdx];
5210  const int idx1 = row_offset + indices[cdx + 1];
5211  const int idx2 = row_offset + indices[cdx + 2];
5212  const int idx3 = row_offset + indices[cdx + 3];
5213 
5214  const __m256 data = _mm256_set_ps(
5215  data_ptr[2 * idx3 + 1],
5216  data_ptr[2 * idx3 + 0],
5217  data_ptr[2 * idx2 + 1],
5218  data_ptr[2 * idx2 + 0],
5219  data_ptr[2 * idx1 + 1],
5220  data_ptr[2 * idx1 + 0],
5221  data_ptr[2 * idx0 + 1],
5222  data_ptr[2 * idx0 + 0]
5223  );
5224 
5225  const __m256 mv_x0 = mv_xy[block_size * rdx + cdx];
5226  const __m256 mv_x1 = mv_xy[block_size * rdx + cdx + 1];
5227  result = _mm256_add_ps(result, _mm256_hadd_ps(_mm256_mul_ps(data, mv_x0), _mm256_mul_ps(data, mv_x1)));
5228 }
5229 
5230 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5231  if (input.cols == 1) {
5232  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5233  }
5234  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5235  throw std::invalid_argument("AVX32 right-apply large-kernel path currently supports 4x4 local kernels only.");
5236  }
5237 
5238  const int qubit_num = (int)std::log2(matrix_size);
5239  int non_targets[64];
5240  int non_target_count = 0;
5241  int block_pattern[4];
5242  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5243 
5244  const Matrix_float transposed = transpose_local_kernel_4x4(unitary);
5245  __m256 mv_xy[16];
5246  construct_mv_xy_vectors32_fixed<2>(transposed, mv_xy);
5247  int indices[4];
5248  QGD_Complex8 out[4];
5249  const int num_blocks = matrix_size >> 2;
5250 
5251  for (int row = 0; row < input.rows; ++row) {
5252  const int row_offset = row * input.stride;
5253 
5254  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5255  int base = 0;
5256  for (int i = 0; i < non_target_count; ++i) {
5257  if (iter_idx & (1ULL << i)) {
5258  base |= (1 << non_targets[i]);
5259  }
5260  }
5261  for (int k = 0; k < 4; ++k) {
5262  indices[k] = base | block_pattern[k];
5263  }
5264 
5265  for (int rdx = 0; rdx < 4; ++rdx) {
5266  __m256 result = _mm256_setzero_ps();
5267  complex_prod_AVX32_fixed_from_right<2>(mv_xy, rdx, 0, indices, input, row_offset, result);
5268 
5269  alignas(32) float acc[8];
5270  _mm256_store_ps(acc, result);
5271  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5272  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5273  }
5274 
5275  for (int rdx = 0; rdx < 4; ++rdx) {
5276  input[row_offset + indices[rdx]] = out[rdx];
5277  }
5278  }
5279  }
5280 }
5281 
5282 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX_OpenMP32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5283  if (input.cols == 1) {
5284  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5285  }
5286  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5287  throw std::invalid_argument("AVX32 right-apply large-kernel path currently supports 4x4 local kernels only.");
5288  }
5289 
5290  const int qubit_num = (int)std::log2(matrix_size);
5291  int non_targets[64];
5292  int non_target_count = 0;
5293  int block_pattern[4];
5294  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5295 
5296  const Matrix_float transposed = transpose_local_kernel_4x4(unitary);
5297  __m256 mv_xy[16];
5298  construct_mv_xy_vectors32_fixed<2>(transposed, mv_xy);
5299  const int num_blocks = matrix_size >> 2;
5300 
5301 #pragma omp parallel for schedule(static)
5302  for (int row = 0; row < input.rows; ++row) {
5303  const int row_offset = row * input.stride;
5304  int indices[4];
5305  QGD_Complex8 out[4];
5306 
5307  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5308  int base = 0;
5309  for (int i = 0; i < non_target_count; ++i) {
5310  if (iter_idx & (1ULL << i)) {
5311  base |= (1 << non_targets[i]);
5312  }
5313  }
5314  for (int k = 0; k < 4; ++k) {
5315  indices[k] = base | block_pattern[k];
5316  }
5317 
5318  for (int rdx = 0; rdx < 4; ++rdx) {
5319  __m256 result = _mm256_setzero_ps();
5320  complex_prod_AVX32_fixed_from_right<2>(mv_xy, rdx, 0, indices, input, row_offset, result);
5321 
5322  alignas(32) float acc[8];
5323  _mm256_store_ps(acc, result);
5324  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5325  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5326  }
5327 
5328  for (int rdx = 0; rdx < 4; ++rdx) {
5329  input[row_offset + indices[rdx]] = out[rdx];
5330  }
5331  }
5332  }
5333 }
5334 
5335 inline void apply_2qbit_kernel_to_matrix_input_from_right_AVX_TBB32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5336  if (input.cols == 1) {
5337  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5338  }
5339  if (involved_qbits.size() != 2 || unitary.rows != 4 || unitary.cols != 4) {
5340  throw std::invalid_argument("AVX32 right-apply large-kernel path currently supports 4x4 local kernels only.");
5341  }
5342 
5343  const int qubit_num = (int)std::log2(matrix_size);
5344  int non_targets[64];
5345  int non_target_count = 0;
5346  int block_pattern[4];
5347  build_2qbit_block_pattern(involved_qbits, non_targets, non_target_count, block_pattern, qubit_num);
5348 
5349  const Matrix_float transposed = transpose_local_kernel_4x4(unitary);
5350  __m256 mv_xy[16];
5351  construct_mv_xy_vectors32_fixed<2>(transposed, mv_xy);
5352  const int num_blocks = matrix_size >> 2;
5353 
5354  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
5355  for (int row = range.begin(); row < range.end(); ++row) {
5356  const int row_offset = row * input.stride;
5357  int indices[4];
5358  QGD_Complex8 out[4];
5359 
5360  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5361  int base = 0;
5362  for (int i = 0; i < non_target_count; ++i) {
5363  if (iter_idx & (1ULL << i)) {
5364  base |= (1 << non_targets[i]);
5365  }
5366  }
5367  for (int k = 0; k < 4; ++k) {
5368  indices[k] = base | block_pattern[k];
5369  }
5370 
5371  for (int rdx = 0; rdx < 4; ++rdx) {
5372  __m256 result = _mm256_setzero_ps();
5373  complex_prod_AVX32_fixed_from_right<2>(mv_xy, rdx, 0, indices, input, row_offset, result);
5374 
5375  alignas(32) float acc[8];
5376  _mm256_store_ps(acc, result);
5377  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5378  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5379  }
5380 
5381  for (int rdx = 0; rdx < 4; ++rdx) {
5382  input[row_offset + indices[rdx]] = out[rdx];
5383  }
5384  }
5385  }
5386  });
5387 }
5388 
5389 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5390  if (input.cols == 1) {
5391  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5392  }
5393 
5394  const int n = static_cast<int>(involved_qbits.size());
5395  const int block_size = 1 << n;
5396  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
5397  throw std::invalid_argument("AVX32 right-apply large-kernel path received an unsupported local kernel size.");
5398  }
5399 
5400  const int qubit_num = (int)std::log2(matrix_size);
5401  std::sort(involved_qbits.begin(), involved_qbits.end());
5402 
5403  std::vector<int> non_targets;
5404  non_targets.reserve(qubit_num - n);
5405  for (int q = 0; q < qubit_num; ++q) {
5406  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
5407  non_targets.push_back(q);
5408  }
5409  }
5410 
5411  std::vector<int> block_pattern(block_size);
5412  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
5413  std::vector<int> indices(block_size);
5414  std::vector<QGD_Complex8> out(block_size);
5415  const int num_blocks = matrix_size >> n;
5416  const Matrix_float transposed = transpose_local_kernel(unitary);
5417  __m256* mv_xy = construct_mv_xy_vectors32(transposed, block_size);
5418 
5419  for (int row = 0; row < input.rows; ++row) {
5420  const int row_offset = row * input.stride;
5421 
5422  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5423  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
5424 
5425  for (int rdx = 0; rdx < block_size; ++rdx) {
5426  __m256 result = _mm256_setzero_ps();
5427 
5428  for (int cdx = 0; cdx < block_size; cdx += 4) {
5429  complex_prod_AVX32_from_right(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
5430  }
5431 
5432  alignas(32) float acc[8];
5433  _mm256_store_ps(acc, result);
5434  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5435  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5436  }
5437 
5438  for (int rdx = 0; rdx < block_size; ++rdx) {
5439  input[row_offset + indices[rdx]] = out[rdx];
5440  }
5441  }
5442  }
5443 
5444  _mm_free(mv_xy);
5445 }
5446 
5447 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX_OpenMP32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5448  if (input.cols == 1) {
5449  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5450  }
5451 
5452  const int n = static_cast<int>(involved_qbits.size());
5453  const int block_size = 1 << n;
5454  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
5455  throw std::invalid_argument("AVX32 right-apply large-kernel path received an unsupported local kernel size.");
5456  }
5457 
5458  const int qubit_num = (int)std::log2(matrix_size);
5459  std::sort(involved_qbits.begin(), involved_qbits.end());
5460 
5461  std::vector<int> non_targets;
5462  non_targets.reserve(qubit_num - n);
5463  for (int q = 0; q < qubit_num; ++q) {
5464  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
5465  non_targets.push_back(q);
5466  }
5467  }
5468 
5469  std::vector<int> block_pattern(block_size);
5470  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
5471  const int num_blocks = matrix_size >> n;
5472  const Matrix_float transposed = transpose_local_kernel(unitary);
5473  __m256* mv_xy = construct_mv_xy_vectors32(transposed, block_size);
5474 
5475 #pragma omp parallel for schedule(static)
5476  for (int row = 0; row < input.rows; ++row) {
5477  const int row_offset = row * input.stride;
5478  std::vector<int> indices(block_size);
5479  std::vector<QGD_Complex8> out(block_size);
5480 
5481  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5482  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
5483 
5484  for (int rdx = 0; rdx < block_size; ++rdx) {
5485  __m256 result = _mm256_setzero_ps();
5486 
5487  for (int cdx = 0; cdx < block_size; cdx += 4) {
5488  complex_prod_AVX32_from_right(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
5489  }
5490 
5491  alignas(32) float acc[8];
5492  _mm256_store_ps(acc, result);
5493  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5494  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5495  }
5496 
5497  for (int rdx = 0; rdx < block_size; ++rdx) {
5498  input[row_offset + indices[rdx]] = out[rdx];
5499  }
5500  }
5501  }
5502 
5503  _mm_free(mv_xy);
5504 }
5505 
5506 inline void apply_nqbit_kernel_to_matrix_input_from_right_AVX_TBB32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5507  if (input.cols == 1) {
5508  throw std::invalid_argument("Right-apply is not supported for column state vectors.");
5509  }
5510 
5511  const int n = static_cast<int>(involved_qbits.size());
5512  const int block_size = 1 << n;
5513  if (n < 2 || n > 5 || unitary.rows != block_size || unitary.cols != block_size) {
5514  throw std::invalid_argument("AVX32 right-apply large-kernel path received an unsupported local kernel size.");
5515  }
5516 
5517  const int qubit_num = (int)std::log2(matrix_size);
5518  std::sort(involved_qbits.begin(), involved_qbits.end());
5519 
5520  std::vector<int> non_targets;
5521  non_targets.reserve(qubit_num - n);
5522  for (int q = 0; q < qubit_num; ++q) {
5523  if (!std::binary_search(involved_qbits.begin(), involved_qbits.end(), q)) {
5524  non_targets.push_back(q);
5525  }
5526  }
5527 
5528  std::vector<int> block_pattern(block_size);
5529  precompute_index_mapping(involved_qbits, non_targets, block_pattern);
5530  const int num_blocks = matrix_size >> n;
5531  const Matrix_float transposed = transpose_local_kernel(unitary);
5532  __m256* mv_xy = construct_mv_xy_vectors32(transposed, block_size);
5533 
5534  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
5535  for (int row = range.begin(); row < range.end(); ++row) {
5536  const int row_offset = row * input.stride;
5537  std::vector<int> indices(block_size);
5538  std::vector<QGD_Complex8> out(block_size);
5539 
5540  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5541  get_block_indices_fast(iter_idx, involved_qbits, non_targets, block_pattern, indices);
5542 
5543  for (int rdx = 0; rdx < block_size; ++rdx) {
5544  __m256 result = _mm256_setzero_ps();
5545 
5546  for (int cdx = 0; cdx < block_size; cdx += 4) {
5547  complex_prod_AVX32_from_right(mv_xy, block_size, rdx, cdx, indices.data(), input, row_offset, result);
5548  }
5549 
5550  alignas(32) float acc[8];
5551  _mm256_store_ps(acc, result);
5552  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5553  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5554  }
5555 
5556  for (int rdx = 0; rdx < block_size; ++rdx) {
5557  input[row_offset + indices[rdx]] = out[rdx];
5558  }
5559  }
5560  }
5561  });
5562 
5563  _mm_free(mv_xy);
5564 }
5565 
5566 } // namespace
5567 
5568 void apply_large_kernel_from_right_AVX(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5569  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5570  apply_2qbit_kernel_to_matrix_input_from_right_AVX_impl(unitary, input, std::move(involved_qbits), matrix_size);
5571  return;
5572  }
5573 
5574  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5575  apply_nqbit_kernel_to_matrix_input_from_right_AVX_impl(unitary, input, std::move(involved_qbits), matrix_size);
5576  return;
5577  }
5578 
5579  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5580 }
5581 
5582 void apply_large_kernel_from_right_AVX_OpenMP(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5583  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5584  apply_2qbit_kernel_to_matrix_input_from_right_AVX_OpenMP_impl(unitary, input, std::move(involved_qbits), matrix_size);
5585  return;
5586  }
5587 
5588  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5589  apply_nqbit_kernel_to_matrix_input_from_right_AVX_OpenMP_impl(unitary, input, std::move(involved_qbits), matrix_size);
5590  return;
5591  }
5592 
5593  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5594 }
5595 
5596 void apply_large_kernel_from_right_AVX_TBB(Matrix& unitary, Matrix& input, std::vector<int> involved_qbits, const int& matrix_size) {
5597  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5598  apply_2qbit_kernel_to_matrix_input_from_right_AVX_TBB_impl(unitary, input, std::move(involved_qbits), matrix_size);
5599  return;
5600  }
5601 
5602  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5603  apply_nqbit_kernel_to_matrix_input_from_right_AVX_TBB_impl(unitary, input, std::move(involved_qbits), matrix_size);
5604  return;
5605  }
5606 
5607  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5608 }
5609 
5610 void apply_large_kernel_from_right_AVX32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5611  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5612  apply_2qbit_kernel_to_matrix_input_from_right_AVX32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5613  return;
5614  }
5615 
5616  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5617  apply_nqbit_kernel_to_matrix_input_from_right_AVX32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5618  return;
5619  }
5620 
5621  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5622 }
5623 
5624 void apply_large_kernel_from_right_AVX_OpenMP32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5625  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5626  apply_2qbit_kernel_to_matrix_input_from_right_AVX_OpenMP32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5627  return;
5628  }
5629 
5630  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5631  apply_nqbit_kernel_to_matrix_input_from_right_AVX_OpenMP32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5632  return;
5633  }
5634 
5635  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5636 }
5637 
5638 void apply_large_kernel_from_right_AVX_TBB32(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5639  if (input.cols != 1 && involved_qbits.size() == 2 && unitary.rows == 4 && unitary.cols == 4) {
5640  apply_2qbit_kernel_to_matrix_input_from_right_AVX_TBB32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5641  return;
5642  }
5643 
5644  if (input.cols != 1 && involved_qbits.size() >= 3 && involved_qbits.size() <= 5) {
5645  apply_nqbit_kernel_to_matrix_input_from_right_AVX_TBB32_impl(unitary, input, std::move(involved_qbits), matrix_size);
5646  return;
5647  }
5648 
5649  apply_large_kernel_from_right(unitary, input, std::move(involved_qbits), matrix_size);
5650 }
5651 
5652 inline __m256* construct_mv_xy_vectors32(const Matrix_float& gate_kernel_unitary, const int& matrix_size) {
5653  __m256* mv_xy = (__m256*) _mm_malloc(sizeof(__m256) * matrix_size * matrix_size, 32);
5654 
5655  for (int rdx = 0; rdx < matrix_size; ++rdx) {
5656  for (int cdx = 0; cdx < matrix_size; cdx += 4) {
5657  mv_xy[rdx * matrix_size + cdx] = _mm256_set_ps(
5658  -gate_kernel_unitary[matrix_size * rdx + cdx + 3].imag,
5659  gate_kernel_unitary[matrix_size * rdx + cdx + 3].real,
5660  -gate_kernel_unitary[matrix_size * rdx + cdx + 2].imag,
5661  gate_kernel_unitary[matrix_size * rdx + cdx + 2].real,
5662  -gate_kernel_unitary[matrix_size * rdx + cdx + 1].imag,
5663  gate_kernel_unitary[matrix_size * rdx + cdx + 1].real,
5664  -gate_kernel_unitary[matrix_size * rdx + cdx].imag,
5665  gate_kernel_unitary[matrix_size * rdx + cdx].real
5666  );
5667 
5668  mv_xy[rdx * matrix_size + cdx + 1] = _mm256_set_ps(
5669  gate_kernel_unitary[matrix_size * rdx + cdx + 3].real,
5670  gate_kernel_unitary[matrix_size * rdx + cdx + 3].imag,
5671  gate_kernel_unitary[matrix_size * rdx + cdx + 2].real,
5672  gate_kernel_unitary[matrix_size * rdx + cdx + 2].imag,
5673  gate_kernel_unitary[matrix_size * rdx + cdx + 1].real,
5674  gate_kernel_unitary[matrix_size * rdx + cdx + 1].imag,
5675  gate_kernel_unitary[matrix_size * rdx + cdx].real,
5676  gate_kernel_unitary[matrix_size * rdx + cdx].imag
5677  );
5678  }
5679  }
5680 
5681  return mv_xy;
5682 }
5683 
5684 template<int n>
5685 inline void construct_mv_xy_vectors32_fixed(const Matrix_float& gate_kernel_unitary, __m256* mv_xy) {
5686  constexpr int block_size = 1 << n;
5687 
5688  for (int rdx = 0; rdx < block_size; ++rdx) {
5689  for (int cdx = 0; cdx < block_size; cdx += 4) {
5690  mv_xy[rdx * block_size + cdx] = _mm256_set_ps(
5691  -gate_kernel_unitary[block_size * rdx + cdx + 3].imag,
5692  gate_kernel_unitary[block_size * rdx + cdx + 3].real,
5693  -gate_kernel_unitary[block_size * rdx + cdx + 2].imag,
5694  gate_kernel_unitary[block_size * rdx + cdx + 2].real,
5695  -gate_kernel_unitary[block_size * rdx + cdx + 1].imag,
5696  gate_kernel_unitary[block_size * rdx + cdx + 1].real,
5697  -gate_kernel_unitary[block_size * rdx + cdx].imag,
5698  gate_kernel_unitary[block_size * rdx + cdx].real
5699  );
5700 
5701  mv_xy[rdx * block_size + cdx + 1] = _mm256_set_ps(
5702  gate_kernel_unitary[block_size * rdx + cdx + 3].real,
5703  gate_kernel_unitary[block_size * rdx + cdx + 3].imag,
5704  gate_kernel_unitary[block_size * rdx + cdx + 2].real,
5705  gate_kernel_unitary[block_size * rdx + cdx + 2].imag,
5706  gate_kernel_unitary[block_size * rdx + cdx + 1].real,
5707  gate_kernel_unitary[block_size * rdx + cdx + 1].imag,
5708  gate_kernel_unitary[block_size * rdx + cdx].real,
5709  gate_kernel_unitary[block_size * rdx + cdx].imag
5710  );
5711  }
5712  }
5713 }
5714 
5715 template<int n>
5716 inline void complex_prod_AVX32_fixed(const __m256* mv_xy, int rdx, int cdx, const int* indices, const Matrix_float& input, int col, __m256& result) {
5717  constexpr int block_size = 1 << n;
5718  const float* data_ptr = (const float*)input.get_data();
5719  const int stride = input.stride;
5720 
5721  const int idx0 = indices[cdx] * stride + col;
5722  const int idx1 = indices[cdx + 1] * stride + col;
5723  const int idx2 = indices[cdx + 2] * stride + col;
5724  const int idx3 = indices[cdx + 3] * stride + col;
5725 
5726  const __m256 data = _mm256_set_ps(
5727  data_ptr[2 * idx3 + 1],
5728  data_ptr[2 * idx3 + 0],
5729  data_ptr[2 * idx2 + 1],
5730  data_ptr[2 * idx2 + 0],
5731  data_ptr[2 * idx1 + 1],
5732  data_ptr[2 * idx1 + 0],
5733  data_ptr[2 * idx0 + 1],
5734  data_ptr[2 * idx0 + 0]
5735  );
5736 
5737  const __m256 mv_x0 = mv_xy[block_size * rdx + cdx];
5738  const __m256 mv_x1 = mv_xy[block_size * rdx + cdx + 1];
5739  result = _mm256_add_ps(result, _mm256_hadd_ps(_mm256_mul_ps(data, mv_x0), _mm256_mul_ps(data, mv_x1)));
5740 }
5741 
5742 template<int n>
5743 void apply_fixed_qbit_unitary_AVX32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5744  constexpr int block_size = 1 << n;
5745  const int qubit_num = (int) std::log2(matrix_size);
5746  const int num_blocks = matrix_size >> n;
5747  std::sort(involved_qbits.begin(), involved_qbits.end());
5748 
5749  int non_targets[64];
5750  int non_target_count = 0;
5751  for (int q = 0; q < qubit_num; ++q) {
5752  bool is_target = false;
5753  for (int target : involved_qbits) {
5754  is_target = is_target || (q == target);
5755  }
5756  if (!is_target) {
5757  non_targets[non_target_count++] = q;
5758  }
5759  }
5760 
5761  int block_pattern[block_size];
5762  for (int k = 0; k < block_size; ++k) {
5763  int idx = 0;
5764  for (int bit = 0; bit < n; ++bit) {
5765  if (k & (1 << bit)) {
5766  idx |= (1 << involved_qbits[bit]);
5767  }
5768  }
5769  block_pattern[k] = idx;
5770  }
5771 
5772  int indices[block_size];
5773  QGD_Complex8 out[block_size];
5774  __m256 mv_xy[block_size * block_size];
5775  construct_mv_xy_vectors32_fixed<n>(gate_kernel_unitary, mv_xy);
5776 
5777  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5778  int base = 0;
5779  for (int i = 0; i < non_target_count; ++i) {
5780  if (iter_idx & (1ULL << i)) {
5781  base |= (1 << non_targets[i]);
5782  }
5783  }
5784  for (int k = 0; k < block_size; ++k) {
5785  indices[k] = base | block_pattern[k];
5786  }
5787 
5788  for (int col = 0; col < input.cols; ++col) {
5789  for (int rdx = 0; rdx < block_size; ++rdx) {
5790  __m256 result = _mm256_setzero_ps();
5791 
5792  for (int cdx = 0; cdx < block_size; cdx += 4) {
5793  complex_prod_AVX32_fixed<n>(mv_xy, rdx, cdx, indices, input, col, result);
5794  }
5795 
5796  alignas(32) float acc[8];
5797  _mm256_store_ps(acc, result);
5798  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5799  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5800  }
5801 
5802  for (int rdx = 0; rdx < block_size; ++rdx) {
5803  input[indices[rdx] * input.stride + col] = out[rdx];
5804  }
5805  }
5806  }
5807 }
5808 
5809 template<int n>
5810 void apply_fixed_qbit_unitary_AVX_OpenMP32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5811  constexpr int block_size = 1 << n;
5812  const int qubit_num = (int) std::log2(matrix_size);
5813  const int num_blocks = matrix_size >> n;
5814  std::sort(involved_qbits.begin(), involved_qbits.end());
5815 
5816  int non_targets[64];
5817  int non_target_count = 0;
5818  for (int q = 0; q < qubit_num; ++q) {
5819  bool is_target = false;
5820  for (int target : involved_qbits) {
5821  is_target = is_target || (q == target);
5822  }
5823  if (!is_target) {
5824  non_targets[non_target_count++] = q;
5825  }
5826  }
5827 
5828  int block_pattern[block_size];
5829  for (int k = 0; k < block_size; ++k) {
5830  int idx = 0;
5831  for (int bit = 0; bit < n; ++bit) {
5832  if (k & (1 << bit)) {
5833  idx |= (1 << involved_qbits[bit]);
5834  }
5835  }
5836  block_pattern[k] = idx;
5837  }
5838 
5839  __m256 mv_xy[block_size * block_size];
5840  construct_mv_xy_vectors32_fixed<n>(gate_kernel_unitary, mv_xy);
5841 
5842 #pragma omp parallel for schedule(static)
5843  for (int iter_idx = 0; iter_idx < num_blocks; ++iter_idx) {
5844  int indices[block_size];
5845  QGD_Complex8 out[block_size];
5846 
5847  int base = 0;
5848  for (int i = 0; i < non_target_count; ++i) {
5849  if (iter_idx & (1ULL << i)) {
5850  base |= (1 << non_targets[i]);
5851  }
5852  }
5853  for (int k = 0; k < block_size; ++k) {
5854  indices[k] = base | block_pattern[k];
5855  }
5856 
5857  for (int col = 0; col < input.cols; ++col) {
5858  for (int rdx = 0; rdx < block_size; ++rdx) {
5859  __m256 result = _mm256_setzero_ps();
5860 
5861  for (int cdx = 0; cdx < block_size; cdx += 4) {
5862  complex_prod_AVX32_fixed<n>(mv_xy, rdx, cdx, indices, input, col, result);
5863  }
5864 
5865  alignas(32) float acc[8];
5866  _mm256_store_ps(acc, result);
5867  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5868  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5869  }
5870 
5871  for (int rdx = 0; rdx < block_size; ++rdx) {
5872  input[indices[rdx] * input.stride + col] = out[rdx];
5873  }
5874  }
5875  }
5876 }
5877 
5878 template<int n>
5879 void apply_fixed_qbit_unitary_AVX_TBB32(Matrix_float& gate_kernel_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5880  constexpr int block_size = 1 << n;
5881  const int qubit_num = (int) std::log2(matrix_size);
5882  const int num_blocks = matrix_size >> n;
5883  std::sort(involved_qbits.begin(), involved_qbits.end());
5884 
5885  int non_targets[64];
5886  int non_target_count = 0;
5887  for (int q = 0; q < qubit_num; ++q) {
5888  bool is_target = false;
5889  for (int target : involved_qbits) {
5890  is_target = is_target || (q == target);
5891  }
5892  if (!is_target) {
5893  non_targets[non_target_count++] = q;
5894  }
5895  }
5896 
5897  int block_pattern[block_size];
5898  for (int k = 0; k < block_size; ++k) {
5899  int idx = 0;
5900  for (int bit = 0; bit < n; ++bit) {
5901  if (k & (1 << bit)) {
5902  idx |= (1 << involved_qbits[bit]);
5903  }
5904  }
5905  block_pattern[k] = idx;
5906  }
5907 
5908  __m256 mv_xy[block_size * block_size];
5909  construct_mv_xy_vectors32_fixed<n>(gate_kernel_unitary, mv_xy);
5910 
5911  tbb::parallel_for(tbb::blocked_range<int>(0, num_blocks, 16), [&](const tbb::blocked_range<int>& range) {
5912  for (int iter_idx = range.begin(); iter_idx < range.end(); ++iter_idx) {
5913  std::array<int, 1 << n> indices;
5914  std::array<QGD_Complex8, 1 << n> out;
5915 
5916  int base = 0;
5917  for (int i = 0; i < non_target_count; ++i) {
5918  if (iter_idx & (1ULL << i)) {
5919  base |= (1 << non_targets[i]);
5920  }
5921  }
5922  for (int k = 0; k < block_size; ++k) {
5923  indices[k] = base | block_pattern[k];
5924  }
5925 
5926  for (int col = 0; col < input.cols; ++col) {
5927  for (int rdx = 0; rdx < block_size; ++rdx) {
5928  __m256 result = _mm256_setzero_ps();
5929 
5930  for (int cdx = 0; cdx < block_size; cdx += 4) {
5931  complex_prod_AVX32_fixed<n>(mv_xy, rdx, cdx, indices.data(), input, col, result);
5932  }
5933 
5934  alignas(32) float acc[8];
5935  _mm256_store_ps(acc, result);
5936  out[rdx].real = acc[0] + acc[1] + acc[4] + acc[5];
5937  out[rdx].imag = acc[2] + acc[3] + acc[6] + acc[7];
5938  }
5939 
5940  for (int rdx = 0; rdx < block_size; ++rdx) {
5941  input[indices[rdx] * input.stride + col] = out[rdx];
5942  }
5943  }
5944  }
5945  });
5946 }
5947 
5948 void apply_2qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float& two_qbit_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5949  apply_fixed_qbit_unitary_AVX32<2>(two_qbit_unitary, input, std::move(involved_qbits), matrix_size);
5950 }
5951 
5952 void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float& two_qbit_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5953  apply_fixed_qbit_unitary_AVX_OpenMP32<2>(two_qbit_unitary, input, std::move(involved_qbits), matrix_size);
5954 }
5955 
5956 void apply_2qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float& two_qbit_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5957  apply_fixed_qbit_unitary_AVX_TBB32<2>(two_qbit_unitary, input, std::move(involved_qbits), matrix_size);
5958 }
5959 
5960 void apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP32(Matrix_float& two_qbit_unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5961  apply_fixed_qbit_unitary_AVX_OpenMP32<2>(two_qbit_unitary, input, std::move(involved_qbits), matrix_size);
5962 }
5963 
5964 void apply_3qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5965  apply_fixed_qbit_unitary_AVX32<3>(unitary, input, std::move(involved_qbits), matrix_size);
5966 }
5967 
5968 void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5969  apply_fixed_qbit_unitary_AVX_OpenMP32<3>(unitary, input, std::move(involved_qbits), matrix_size);
5970 }
5971 
5972 void apply_3qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5973  apply_fixed_qbit_unitary_AVX_TBB32<3>(unitary, input, std::move(involved_qbits), matrix_size);
5974 }
5975 
5976 void apply_4qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5977  apply_fixed_qbit_unitary_AVX32<4>(unitary, input, std::move(involved_qbits), matrix_size);
5978 }
5979 
5980 void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5981  apply_fixed_qbit_unitary_AVX_OpenMP32<4>(unitary, input, std::move(involved_qbits), matrix_size);
5982 }
5983 
5984 void apply_4qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5985  apply_fixed_qbit_unitary_AVX_TBB32<4>(unitary, input, std::move(involved_qbits), matrix_size);
5986 }
5987 
5988 void apply_5qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5989  apply_fixed_qbit_unitary_AVX32<5>(unitary, input, std::move(involved_qbits), matrix_size);
5990 }
5991 
5992 void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5993  apply_fixed_qbit_unitary_AVX_OpenMP32<5>(unitary, input, std::move(involved_qbits), matrix_size);
5994 }
5995 
5996 void apply_5qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float& unitary, Matrix_float& input, std::vector<int> involved_qbits, const int& matrix_size) {
5997  apply_fixed_qbit_unitary_AVX_TBB32<5>(unitary, input, std::move(involved_qbits), matrix_size);
5998 }
5999 
6000 void apply_crot_kernel_to_matrix_input_AVX_parallel32(Matrix_float& u3_1qbit1, Matrix_float& u3_1qbit2, Matrix_float& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
6001  const int index_step_target = 1 << target_qbit;
6002  const int parallel_outer_cycles = matrix_size / (index_step_target << 1);
6003 
6004  int outer_grain_size;
6005  if (index_step_target <= 2) {
6006  outer_grain_size = 32;
6007  }
6008  else if (index_step_target <= 4) {
6009  outer_grain_size = 16;
6010  }
6011  else if (index_step_target <= 8) {
6012  outer_grain_size = 8;
6013  }
6014  else {
6015  outer_grain_size = 1;
6016  }
6017 
6018  tbb::parallel_for(tbb::blocked_range<int>(0, parallel_outer_cycles, outer_grain_size), [&](const tbb::blocked_range<int>& range) {
6019  for (int outer_idx = range.begin(); outer_idx < range.end(); ++outer_idx) {
6020  const int current_idx = outer_idx * (index_step_target << 1);
6021  const int current_idx_pair = current_idx + index_step_target;
6022 
6023  for (int idx = 0; idx < index_step_target; ++idx) {
6024  const int current_idx_loc = current_idx + idx;
6025  const int current_idx_pair_loc = current_idx_pair + idx;
6026  const int row_offset = current_idx_loc * input.stride;
6027  const int row_offset_pair = current_idx_pair_loc * input.stride;
6028  const Matrix_float& gate = ((current_idx_loc >> control_qbit) & 1) ? u3_1qbit1 : u3_1qbit2;
6029 
6030  for (int col = 0; col < input.cols; ++col) {
6031  const int index = row_offset + col;
6032  const int index_pair = row_offset_pair + col;
6033 
6034  QGD_Complex8 a = input[index];
6035  QGD_Complex8 b = input[index_pair];
6036 
6037  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6038  real = std::fma(gate[1].real, b.real, real);
6039  real = std::fma(-gate[1].imag, b.imag, real);
6040  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6041  imag = std::fma(gate[1].real, b.imag, imag);
6042  imag = std::fma(gate[1].imag, b.real, imag);
6043  input[index].real = real;
6044  input[index].imag = imag;
6045 
6046  real = std::fma(gate[2].real, a.real, -gate[2].imag * a.imag);
6047  real = std::fma(gate[3].real, b.real, real);
6048  real = std::fma(-gate[3].imag, b.imag, real);
6049  imag = std::fma(gate[2].real, a.imag, gate[2].imag * a.real);
6050  imag = std::fma(gate[3].real, b.imag, imag);
6051  imag = std::fma(gate[3].imag, b.real, imag);
6052  input[index_pair].real = real;
6053  input[index_pair].imag = imag;
6054  }
6055  }
6056  }
6057  });
6058 }
6059 
6060 void apply_crot_kernel_to_matrix_input_from_right_AVX_parallel32(Matrix_float& u3_1qbit1, Matrix_float& u3_1qbit2, Matrix_float& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
6061  const int index_step_target = 1 << target_qbit;
6062 
6063  auto cmul_ps = [](__m256 ar, __m256 ai, __m256 x) {
6064  const __m256 swapped = _mm256_permute_ps(x, 0xB1);
6065  return _mm256_fmaddsub_ps(ar, x, _mm256_mul_ps(ai, swapped));
6066  };
6067 
6068  tbb::parallel_for(tbb::blocked_range<int>(0, input.rows, 16), [&](const tbb::blocked_range<int>& range) {
6069  for (int row_idx = range.begin(); row_idx < range.end(); ++row_idx) {
6070  const int row_offset = row_idx * input.stride;
6071  int current_idx = 0;
6072  int current_idx_pair = index_step_target;
6073 
6074  while (current_idx_pair < input.cols) {
6075  const bool mixed = (control_qbit >= 0 && control_qbit < target_qbit);
6076 
6077  if (!mixed) {
6078  Matrix_float& gate = ((control_qbit < 0) || ((current_idx >> control_qbit) & 1)) ? u3_1qbit1 : u3_1qbit2;
6079  const __m256 u00r = _mm256_set1_ps(gate[0].real);
6080  const __m256 u00i = _mm256_set1_ps(gate[0].imag);
6081  const __m256 u01r = _mm256_set1_ps(gate[1].real);
6082  const __m256 u01i = _mm256_set1_ps(gate[1].imag);
6083  const __m256 u10r = _mm256_set1_ps(gate[2].real);
6084  const __m256 u10i = _mm256_set1_ps(gate[2].imag);
6085  const __m256 u11r = _mm256_set1_ps(gate[3].real);
6086  const __m256 u11i = _mm256_set1_ps(gate[3].imag);
6087  float* element = (float*)input.get_data() + 2 * (row_offset + current_idx);
6088  float* element_pair = (float*)input.get_data() + 2 * (row_offset + current_idx_pair);
6089  int col_idx = 0;
6090  const int avx_limit = 2 * index_step_target - 8;
6091 
6092  for (; col_idx <= avx_limit; col_idx += 8) {
6093  const __m256 e = _mm256_loadu_ps(element + col_idx);
6094  const __m256 p = _mm256_loadu_ps(element_pair + col_idx);
6095  const __m256 out0 = _mm256_add_ps(cmul_ps(u00r, u00i, e), cmul_ps(u10r, u10i, p));
6096  const __m256 out1 = _mm256_add_ps(cmul_ps(u01r, u01i, e), cmul_ps(u11r, u11i, p));
6097  _mm256_storeu_ps(element + col_idx, out0);
6098  _mm256_storeu_ps(element_pair + col_idx, out1);
6099  }
6100 
6101  for (int c = col_idx / 2; c < index_step_target; ++c) {
6102  const int index = row_offset + current_idx + c;
6103  const int index_pair = row_offset + current_idx_pair + c;
6104  QGD_Complex8 a = input[index];
6105  QGD_Complex8 b = input[index_pair];
6106  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6107  real = std::fma(gate[2].real, b.real, real);
6108  real = std::fma(-gate[2].imag, b.imag, real);
6109  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6110  imag = std::fma(gate[2].real, b.imag, imag);
6111  imag = std::fma(gate[2].imag, b.real, imag);
6112  input[index].real = real;
6113  input[index].imag = imag;
6114  real = std::fma(gate[1].real, a.real, -gate[1].imag * a.imag);
6115  real = std::fma(gate[3].real, b.real, real);
6116  real = std::fma(-gate[3].imag, b.imag, real);
6117  imag = std::fma(gate[1].real, a.imag, gate[1].imag * a.real);
6118  imag = std::fma(gate[3].real, b.imag, imag);
6119  imag = std::fma(gate[3].imag, b.real, imag);
6120  input[index_pair].real = real;
6121  input[index_pair].imag = imag;
6122  }
6123  } else {
6124  for (int idx = 0; idx < index_step_target; ++idx) {
6125  const int current_idx_loc = current_idx + idx;
6126  const int current_idx_pair_loc = current_idx_pair + idx;
6127  Matrix_float& gate = ((current_idx_loc >> control_qbit) & 1) ? u3_1qbit1 : u3_1qbit2;
6128  const int index = row_offset + current_idx_loc;
6129  const int index_pair = row_offset + current_idx_pair_loc;
6130  QGD_Complex8 a = input[index];
6131  QGD_Complex8 b = input[index_pair];
6132  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6133  real = std::fma(gate[2].real, b.real, real);
6134  real = std::fma(-gate[2].imag, b.imag, real);
6135  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6136  imag = std::fma(gate[2].real, b.imag, imag);
6137  imag = std::fma(gate[2].imag, b.real, imag);
6138  input[index].real = real;
6139  input[index].imag = imag;
6140  real = std::fma(gate[1].real, a.real, -gate[1].imag * a.imag);
6141  real = std::fma(gate[3].real, b.real, real);
6142  real = std::fma(-gate[3].imag, b.imag, real);
6143  imag = std::fma(gate[1].real, a.imag, gate[1].imag * a.real);
6144  imag = std::fma(gate[3].real, b.imag, imag);
6145  imag = std::fma(gate[3].imag, b.real, imag);
6146  input[index_pair].real = real;
6147  input[index_pair].imag = imag;
6148  }
6149  }
6150 
6151  current_idx += (index_step_target << 1);
6152  current_idx_pair += (index_step_target << 1);
6153  }
6154  }
6155  });
6156 
6157  (void)matrix_size;
6158 }
6159 
6160 void apply_crot_kernel_to_matrix_input_AVX32(Matrix_float& u3_1qbit1, Matrix_float& u3_qbit2, Matrix_float& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
6161  const int index_step_target = 1 << target_qbit;
6162  int current_idx = 0;
6163 
6164  for (int current_idx_pair = current_idx + index_step_target; current_idx_pair < matrix_size; current_idx_pair += (index_step_target << 1)) {
6165  for (int idx = 0; idx < index_step_target; ++idx) {
6166  const int current_idx_loc = current_idx + idx;
6167  const int current_idx_pair_loc = current_idx_pair + idx;
6168  const int row_offset = current_idx_loc * input.stride;
6169  const int row_offset_pair = current_idx_pair_loc * input.stride;
6170  const Matrix_float& gate = ((current_idx_loc >> control_qbit) & 1) ? u3_1qbit1 : u3_qbit2;
6171 
6172  for (int col = 0; col < input.cols; ++col) {
6173  const int index = row_offset + col;
6174  const int index_pair = row_offset_pair + col;
6175 
6176  QGD_Complex8 a = input[index];
6177  QGD_Complex8 b = input[index_pair];
6178 
6179  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6180  real = std::fma(gate[1].real, b.real, real);
6181  real = std::fma(-gate[1].imag, b.imag, real);
6182  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6183  imag = std::fma(gate[1].real, b.imag, imag);
6184  imag = std::fma(gate[1].imag, b.real, imag);
6185  input[index].real = real;
6186  input[index].imag = imag;
6187 
6188  real = std::fma(gate[2].real, a.real, -gate[2].imag * a.imag);
6189  real = std::fma(gate[3].real, b.real, real);
6190  real = std::fma(-gate[3].imag, b.imag, real);
6191  imag = std::fma(gate[2].real, a.imag, gate[2].imag * a.real);
6192  imag = std::fma(gate[3].real, b.imag, imag);
6193  imag = std::fma(gate[3].imag, b.real, imag);
6194  input[index_pair].real = real;
6195  input[index_pair].imag = imag;
6196  }
6197  }
6198 
6199  current_idx += (index_step_target << 1);
6200  }
6201 }
6202 
6203 void apply_crot_kernel_to_matrix_input_from_right_AVX32(Matrix_float& u3_1qbit1, Matrix_float& u3_qbit2, Matrix_float& input, const int& target_qbit, const int& control_qbit, const int& matrix_size) {
6204  const int index_step_target = 1 << target_qbit;
6205 
6206  auto cmul_ps = [](__m256 ar, __m256 ai, __m256 x) {
6207  const __m256 swapped = _mm256_permute_ps(x, 0xB1);
6208  return _mm256_fmaddsub_ps(ar, x, _mm256_mul_ps(ai, swapped));
6209  };
6210 
6211  int row_idx = 0;
6212  for (; row_idx < input.rows; ++row_idx) {
6213  const int row_offset = row_idx * input.stride;
6214  int current_idx = 0;
6215  int current_idx_pair = index_step_target;
6216 
6217  while (current_idx_pair < input.cols) {
6218  const bool mixed = (control_qbit >= 0 && control_qbit < target_qbit);
6219 
6220  if (!mixed) {
6221  Matrix_float& gate = ((control_qbit < 0) || ((current_idx >> control_qbit) & 1)) ? u3_1qbit1 : u3_qbit2;
6222  const __m256 u00r = _mm256_set1_ps(gate[0].real);
6223  const __m256 u00i = _mm256_set1_ps(gate[0].imag);
6224  const __m256 u01r = _mm256_set1_ps(gate[1].real);
6225  const __m256 u01i = _mm256_set1_ps(gate[1].imag);
6226  const __m256 u10r = _mm256_set1_ps(gate[2].real);
6227  const __m256 u10i = _mm256_set1_ps(gate[2].imag);
6228  const __m256 u11r = _mm256_set1_ps(gate[3].real);
6229  const __m256 u11i = _mm256_set1_ps(gate[3].imag);
6230  float* element = (float*)input.get_data() + 2 * (row_offset + current_idx);
6231  float* element_pair = (float*)input.get_data() + 2 * (row_offset + current_idx_pair);
6232  int col_idx = 0;
6233  const int avx_limit = 2 * index_step_target - 8;
6234 
6235  for (; col_idx <= avx_limit; col_idx += 8) {
6236  const __m256 e = _mm256_loadu_ps(element + col_idx);
6237  const __m256 p = _mm256_loadu_ps(element_pair + col_idx);
6238  const __m256 out0 = _mm256_add_ps(cmul_ps(u00r, u00i, e), cmul_ps(u10r, u10i, p));
6239  const __m256 out1 = _mm256_add_ps(cmul_ps(u01r, u01i, e), cmul_ps(u11r, u11i, p));
6240  _mm256_storeu_ps(element + col_idx, out0);
6241  _mm256_storeu_ps(element_pair + col_idx, out1);
6242  }
6243 
6244  for (int c = col_idx / 2; c < index_step_target; ++c) {
6245  const int index = row_offset + current_idx + c;
6246  const int index_pair = row_offset + current_idx_pair + c;
6247  QGD_Complex8 a = input[index];
6248  QGD_Complex8 b = input[index_pair];
6249  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6250  real = std::fma(gate[2].real, b.real, real);
6251  real = std::fma(-gate[2].imag, b.imag, real);
6252  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6253  imag = std::fma(gate[2].real, b.imag, imag);
6254  imag = std::fma(gate[2].imag, b.real, imag);
6255  input[index].real = real;
6256  input[index].imag = imag;
6257  real = std::fma(gate[1].real, a.real, -gate[1].imag * a.imag);
6258  real = std::fma(gate[3].real, b.real, real);
6259  real = std::fma(-gate[3].imag, b.imag, real);
6260  imag = std::fma(gate[1].real, a.imag, gate[1].imag * a.real);
6261  imag = std::fma(gate[3].real, b.imag, imag);
6262  imag = std::fma(gate[3].imag, b.real, imag);
6263  input[index_pair].real = real;
6264  input[index_pair].imag = imag;
6265  }
6266  } else {
6267  for (int idx = 0; idx < index_step_target; ++idx) {
6268  const int current_idx_loc = current_idx + idx;
6269  const int current_idx_pair_loc = current_idx_pair + idx;
6270  Matrix_float& gate = ((current_idx_loc >> control_qbit) & 1) ? u3_1qbit1 : u3_qbit2;
6271  const int index = row_offset + current_idx_loc;
6272  const int index_pair = row_offset + current_idx_pair_loc;
6273  QGD_Complex8 a = input[index];
6274  QGD_Complex8 b = input[index_pair];
6275  float real = std::fma(gate[0].real, a.real, -gate[0].imag * a.imag);
6276  real = std::fma(gate[2].real, b.real, real);
6277  real = std::fma(-gate[2].imag, b.imag, real);
6278  float imag = std::fma(gate[0].real, a.imag, gate[0].imag * a.real);
6279  imag = std::fma(gate[2].real, b.imag, imag);
6280  imag = std::fma(gate[2].imag, b.real, imag);
6281  input[index].real = real;
6282  input[index].imag = imag;
6283  real = std::fma(gate[1].real, a.real, -gate[1].imag * a.imag);
6284  real = std::fma(gate[3].real, b.real, real);
6285  real = std::fma(-gate[3].imag, b.imag, real);
6286  imag = std::fma(gate[1].real, a.imag, gate[1].imag * a.real);
6287  imag = std::fma(gate[3].real, b.imag, imag);
6288  imag = std::fma(gate[3].imag, b.real, imag);
6289  input[index_pair].real = real;
6290  input[index_pair].imag = imag;
6291  }
6292  }
6293 
6294  current_idx += (index_step_target << 1);
6295  current_idx_pair += (index_step_target << 1);
6296  }
6297  }
6298 
6299  (void)matrix_size;
6300 }
apply_2qbit_kernel_to_state_vector_input_AVX
void apply_2qbit_kernel_to_state_vector_input_AVX(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1125
apply_kernel_to_state_vector_input_AVX.h
apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32
void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1213
apply_large_kernel_from_right_AVX32
void apply_large_kernel_from_right_AVX32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5610
apply_3qbit_kernel_to_state_vector_input_AVX32
void apply_3qbit_kernel_to_state_vector_input_AVX32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1185
apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32
void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1177
tests.decomposition.test_optmization_problem_combined.real
real
Definition: test_optmization_problem_combined.py:67
apply_large_kernel_from_right_AVX_TBB32
void apply_large_kernel_from_right_AVX_TBB32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5638
apply_large_kernel_from_right_AVX_OpenMP
void apply_large_kernel_from_right_AVX_OpenMP(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5582
apply_large_kernel_to_input_AVX_TBB
void apply_large_kernel_to_input_AVX_TBB(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply multi-qubit gate kernel to an input matrix using AVX optimization and TBB parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:373
apply_5qbit_kernel_to_state_vector_input_AVX
void apply_5qbit_kernel_to_state_vector_input_AVX(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1161
apply_3qbit_kernel_to_state_vector_input_AVX_TBB
void apply_3qbit_kernel_to_state_vector_input_AVX_TBB(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1145
apply_5qbit_kernel_to_state_vector_input_AVX32_impl
void apply_5qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5988
apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP
void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1141
apply_large_kernel_from_right_AVX_TBB
void apply_large_kernel_from_right_AVX_TBB(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5596
apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP
void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1153
construct_mv_xy_vectors32_fixed
void construct_mv_xy_vectors32_fixed(const Matrix_float &gate_kernel_unitary, __m256 *mv_xy)
Definition: apply_large_kernel_to_input_AVX.cpp:5685
construct_mv_xy_vectors32
__m256 * construct_mv_xy_vectors32(const Matrix_float &gate_kernel_unitary, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5652
apply_4qbit_kernel_to_state_vector_input_AVX
void apply_4qbit_kernel_to_state_vector_input_AVX(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1149
apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl
void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5992
apply_crot_kernel_to_matrix_input_from_right_AVX32
void apply_crot_kernel_to_matrix_input_from_right_AVX32(Matrix_float &u3_1qbit1, Matrix_float &u3_qbit2, Matrix_float &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:6203
apply_4qbit_kernel_to_state_vector_input_AVX_TBB32
void apply_4qbit_kernel_to_state_vector_input_AVX_TBB32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1205
apply_2qbit_kernel_to_state_vector_input_AVX_TBB32_impl
void apply_2qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5956
matrix_base::stride
int stride
The column stride of the array. (The array elements in one row are a_0, a_1, ... a_{cols-1}, 0, 0, 0, 0. The number of zeros is stride-cols)
Definition: matrix_base.hpp:46
apply_crot_kernel_to_matrix_input_AVX_parallel32
void apply_crot_kernel_to_matrix_input_AVX_parallel32(Matrix_float &u3_1qbit1, Matrix_float &u3_1qbit2, Matrix_float &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:6000
QGD_Complex8::real
float real
real part
Definition: QGDTypes.h:47
apply_fixed_qbit_unitary_AVX_OpenMP
void apply_fixed_qbit_unitary_AVX_OpenMP(Matrix &gate_kernel_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4595
apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP_impl
void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply three-qubit gate kernel to a state vector using AVX optimization and OpenMP parallelization...
Definition: apply_large_kernel_to_input_AVX.cpp:2095
apply_4qbit_kernel_to_state_vector_input_AVX32
void apply_4qbit_kernel_to_state_vector_input_AVX32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1197
QGD_Complex8
Structure type representing single-precision complex numbers.
Definition: QGDTypes.h:46
apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP
void apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply two-qubit gate kernel to an input matrix using AVX optimization and OpenMP parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:3520
apply_3qbit_kernel_to_state_vector_input_AVX_impl
void apply_3qbit_kernel_to_state_vector_input_AVX_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply a 3-qubit quantum gate (unitary matrix) to a state vector using AVX intrinsics.
Definition: apply_large_kernel_to_input_AVX.cpp:849
apply_2qbit_kernel_to_state_vector_input_AVX_TBB_impl
void apply_2qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply two-qubit gate kernel to a state vector using AVX optimization and TBB parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:1825
construct_mv_xy_vectors_fixed
void construct_mv_xy_vectors_fixed(const Matrix &gate_kernel_unitary, __m256d *mv_xy)
Definition: apply_large_kernel_to_input_AVX.cpp:4483
apply_large_kernel_to_input
void apply_large_kernel_to_input(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input.cpp:574
apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP_impl
void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply two-qubit gate kernel to a state vector using AVX optimization and OpenMP parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:1553
apply_2qbit_kernel_to_state_vector_input_AVX32
void apply_2qbit_kernel_to_state_vector_input_AVX32(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1173
example.data
data
load the unitary from file
Definition: example.py:51
matrix_base::ensure_aligned
void ensure_aligned()
Definition: matrix_base.hpp:448
mult
QGD_Complex16 mult(QGD_Complex16 &a, QGD_Complex16 &b)
Call to calculate the product of two complex scalars.
Definition: common.cpp:298
QGD_Complex8::imag
float imag
imaginary part
Definition: QGDTypes.h:48
apply_2qbit_kernel_to_state_vector_input_AVX_impl
void apply_2qbit_kernel_to_state_vector_input_AVX_impl(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply two-qubit gate kernel to a state vector using AVX optimization.
Definition: apply_large_kernel_to_input_AVX.cpp:516
apply_fixed_qbit_unitary_AVX_TBB32
void apply_fixed_qbit_unitary_AVX_TBB32(Matrix_float &gate_kernel_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5879
apply_4qbit_kernel_to_state_vector_input_AVX_TBB
void apply_4qbit_kernel_to_state_vector_input_AVX_TBB(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1157
apply_crot_kernel_to_matrix_input_AVX
void apply_crot_kernel_to_matrix_input_AVX(Matrix &u3_1qbit1, Matrix &u3_1qbit2, Matrix &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Call to apply crot gate kernel on an input matrix using AVX.
Definition: apply_large_kernel_to_input_AVX.cpp:3617
apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP_impl
void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply four-qubit gate kernel to a state vector using AVX optimization and OpenMP parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:2443
GQML_test.x
x
Definition: GQML_test.py:52
matrix_base::get_data
scalar * get_data() const
Call to get the pointer to the stored data.
Definition: matrix_base.hpp:285
apply_2qbit_kernel_to_state_vector_input_AVX32_impl
void apply_2qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5948
COMPUTE_3QBIT_ROW
#define COMPUTE_3QBIT_ROW(row_letter, base0, base1, base2, base3, base4, base5, base6, base7)
Definition: apply_large_kernel_to_input_AVX.cpp:808
construct_mv_xy_vectors
__m256d * construct_mv_xy_vectors(const Matrix &gate_kernel_unitary, const int &matrix_size)
Write the computed block back to the input matrix.
Definition: apply_large_kernel_to_input_AVX.cpp:481
apply_fixed_qbit_unitary_AVX_TBB
void apply_fixed_qbit_unitary_AVX_TBB(Matrix &gate_kernel_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4665
apply_large_kernel_to_input_AVX
void apply_large_kernel_to_input_AVX(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply multi-qubit gate kernel to an input matrix using AVX optimization.
Definition: apply_large_kernel_to_input_AVX.cpp:90
apply_5qbit_kernel_to_state_vector_input_AVX_impl
void apply_5qbit_kernel_to_state_vector_input_AVX_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply five-qubit gate kernel to a state vector using AVX optimization.
Definition: apply_large_kernel_to_input_AVX.cpp:1216
apply_4qbit_kernel_to_state_vector_input_AVX32_impl
void apply_4qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5976
apply_5qbit_kernel_to_state_vector_input_AVX_TBB32_impl
void apply_5qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5996
apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32
void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1201
matrix_base::rows
int rows
The number of rows.
Definition: matrix_base.hpp:42
matrix_base::cols
int cols
The number of columns.
Definition: matrix_base.hpp:44
example.matrix_size
matrix_size
[load Umtx]
Definition: example.py:58
apply_5qbit_kernel_to_state_vector_input_AVX_TBB
void apply_5qbit_kernel_to_state_vector_input_AVX_TBB(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1169
apply_5qbit_kernel_to_state_vector_input_AVX_TBB32
void apply_5qbit_kernel_to_state_vector_input_AVX_TBB32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1217
QC_sim_benchmark.target_qbit
target_qbit
Definition: QC_sim_benchmark.py:86
apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl
void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5952
apply_crot_kernel_to_matrix_input_AVX32
void apply_crot_kernel_to_matrix_input_AVX32(Matrix_float &u3_1qbit1, Matrix_float &u3_qbit2, Matrix_float &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:6160
apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl
void apply_4qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5980
16_qubit_trained_circuit_VQE.k
k
Definition: 16_qubit_trained_circuit_VQE.py:150
complex_prod_AVX_fixed
void complex_prod_AVX_fixed(const __m256d *mv_xy, int rdx, int cdx, const int *indices, const Matrix &input, int col, __m256d &result)
Definition: apply_large_kernel_to_input_AVX.cpp:4506
example_get_circuit_unitary.unitary
unitary
Definition: example_get_circuit_unitary.py:105
complex_prod_AVX32_fixed
void complex_prod_AVX32_fixed(const __m256 *mv_xy, int rdx, int cdx, const int *indices, const Matrix_float &input, int col, __m256 &result)
Definition: apply_large_kernel_to_input_AVX.cpp:5716
tests.VQE.test_shot_noise_measurement.n
n
Definition: test_shot_noise_measurement.py:316
apply_crot_kernel_to_matrix_input_from_right_AVX_parallel32
void apply_crot_kernel_to_matrix_input_from_right_AVX_parallel32(Matrix_float &u3_1qbit1, Matrix_float &u3_1qbit2, Matrix_float &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:6060
QGD_Complex16
Structure type representing complex numbers in the SQUANDER package.
Definition: QGDTypes.h:38
CREATE_MATRIX_VECTOR
#define CREATE_MATRIX_VECTOR(base_idx)
Definition: apply_large_kernel_to_input_AVX.cpp:804
Matrix::copy
Matrix copy() const
Call to create a copy of the matrix.
Definition: matrix.h:57
apply_fixed_qbit_unitary_AVX_OpenMP32
void apply_fixed_qbit_unitary_AVX_OpenMP32(Matrix_float &gate_kernel_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5810
Matrix
Double-precision complex matrix (float64).
Definition: matrix.h:38
CREATE_MATRIX_VECTOR_CONSECUTIVE
#define CREATE_MATRIX_VECTOR_CONSECUTIVE(base_idx)
Definition: apply_large_kernel_to_input_AVX.cpp:779
apply_large_kernel_to_input_AVX_TBB32
void apply_large_kernel_to_input_AVX_TBB32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4433
apply_3qbit_kernel_to_state_vector_input_AVX32_impl
void apply_3qbit_kernel_to_state_vector_input_AVX32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5964
apply_large_kernel_to_input_AVX_OpenMP32
void apply_large_kernel_to_input_AVX_OpenMP32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4389
apply_large_kernel_from_right_AVX_OpenMP32
void apply_large_kernel_from_right_AVX_OpenMP32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5624
precompute_index_mapping
void precompute_index_mapping(const std::vector< int > &target_qubits, const std::vector< int > &non_targets, std::vector< int > &block_pattern)
Precompute the index mapping for target and non-target qubits.
Definition: apply_large_kernel_to_input_AVX.cpp:426
apply_large_kernel_to_input_AVX32
void apply_large_kernel_to_input_AVX32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4345
apply_crot_kernel_to_matrix_input_AVX_parallel
void apply_crot_kernel_to_matrix_input_AVX_parallel(Matrix &u3_1qbit1, Matrix &u3_1qbit2, Matrix &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Call to apply crot gate kernel on an input matrix using AVX and TBB.
Definition: apply_large_kernel_to_input_AVX.cpp:3836
apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP_impl
void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply five-qubit gate kernel to a state vector using AVX optimization and OpenMP parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:2842
apply_large_kernel_to_input_AVX.h
Header file for AVX-optimized implementations for applying multi-qubit gate kernels to quantum state ...
apply_3qbit_kernel_to_state_vector_input_AVX_TBB32_impl
void apply_3qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5972
Matrix_float
Single-precision complex matrix (float32).
Definition: matrix_float.h:41
wide_circuit_optimization.f
f
Definition: wide_circuit_optimization.py:171
apply_fixed_qbit_unitary_AVX
void apply_fixed_qbit_unitary_AVX(Matrix &gate_kernel_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4527
apply_2qbit_kernel_to_state_vector_input_AVX_TBB32
void apply_2qbit_kernel_to_state_vector_input_AVX_TBB32(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1181
complex_mult_AVX
__m256d complex_mult_AVX(__m256d input_vec, __m256d unitary_row_vec, __m256d neg)
Perform complex multiplication using AVX intrinsics.
Definition: apply_large_kernel_to_input_AVX.cpp:62
apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP
void apply_2qbit_kernel_to_state_vector_input_AVX_OpenMP(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1129
apply_2qbit_kernel_to_state_vector_input_AVX_TBB
void apply_2qbit_kernel_to_state_vector_input_AVX_TBB(Matrix &two_qbit_unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1133
apply_large_kernel_from_right_AVX
void apply_large_kernel_from_right_AVX(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5568
apply_3qbit_kernel_to_state_vector_input_AVX_TBB_impl
void apply_3qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply three-qubit gate kernel to a state vector using AVX optimization and TBB parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:2268
apply_5qbit_kernel_to_state_vector_input_AVX_TBB_impl
void apply_5qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply five-qubit gate kernel to a state vector using AVX optimization and TBB parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:3179
apply_4qbit_kernel_to_state_vector_input_AVX_impl
void apply_4qbit_kernel_to_state_vector_input_AVX_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply four-qubit gate kernel to a state vector using AVX optimization.
Definition: apply_large_kernel_to_input_AVX.cpp:1013
apply_crot_kernel_to_matrix_input_from_right_AVX_parallel
void apply_crot_kernel_to_matrix_input_from_right_AVX_parallel(Matrix &u3_1qbit1, Matrix &u3_1qbit2, Matrix &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4217
apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP32
void apply_2qbit_kernel_to_matrix_input_parallel_AVX_OpenMP32(Matrix_float &two_qbit_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5960
COMPUTE_3QBIT_ROW_CONSECUTIVE
#define COMPUTE_3QBIT_ROW_CONSECUTIVE(row_letter, mv00, mv20, mv40, mv60)
Definition: apply_large_kernel_to_input_AVX.cpp:783
get_block_indices_fast
void get_block_indices_fast(int iter_idx, const std::vector< int > &target_qubits, const std::vector< int > &non_targets, const std::vector< int > &block_pattern, std::vector< int > &indices)
Efficiently compute the indices for a given block using precomputed patterns.
Definition: apply_large_kernel_to_input_AVX.cpp:451
apply_large_kernel_to_input_AVX_OpenMP
void apply_large_kernel_to_input_AVX_OpenMP(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply multi-qubit gate kernel to an input matrix using AVX optimization and OpenMP parallelization...
Definition: apply_large_kernel_to_input_AVX.cpp:145
apply_4qbit_kernel_to_state_vector_input_AVX_TBB32_impl
void apply_4qbit_kernel_to_state_vector_input_AVX_TBB32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5984
QGD_Complex16::real
double real
the real part of a complex number
Definition: QGDTypes.h:40
wide_circuit_optimization.results
dictionary results
Definition: wide_circuit_optimization.py:97
get_AVX_vector
__m256d get_AVX_vector(double *element_outer, double *element_inner)
Helper function to load and prepare AVX vectors with outer and inner elements for complex multiplicat...
Definition: apply_large_kernel_to_input_AVX.cpp:43
apply_3qbit_kernel_to_state_vector_input_AVX_TBB32
void apply_3qbit_kernel_to_state_vector_input_AVX_TBB32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1193
apply_4qbit_kernel_to_state_vector_input_AVX_TBB_impl
void apply_4qbit_kernel_to_state_vector_input_AVX_TBB_impl(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Apply four-qubit gate kernel to a state vector using AVX optimization and TBB parallelization.
Definition: apply_large_kernel_to_input_AVX.cpp:2640
apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32
void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1189
apply_large_kernel_from_right
void apply_large_kernel_from_right(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input.cpp:582
apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl
void apply_3qbit_kernel_to_state_vector_input_AVX_OpenMP32_impl(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5968
Matrix_float::copy
Matrix_float copy() const
Call to create a copy of the matrix.
Definition: matrix_float.h:60
apply_fixed_qbit_unitary_AVX32
void apply_fixed_qbit_unitary_AVX32(Matrix_float &gate_kernel_unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:5743
apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP
void apply_5qbit_kernel_to_state_vector_input_AVX_OpenMP(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1165
apply_3qbit_kernel_to_state_vector_input_AVX
void apply_3qbit_kernel_to_state_vector_input_AVX(Matrix &unitary, Matrix &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1137
apply_5qbit_kernel_to_state_vector_input_AVX32
void apply_5qbit_kernel_to_state_vector_input_AVX32(Matrix_float &unitary, Matrix_float &input, std::vector< int > involved_qbits, const int &matrix_size)
Definition: apply_kernel_to_state_vector_input_AVX.cpp:1209
QGD_Complex16::imag
double imag
the imaginary part of a complex number
Definition: QGDTypes.h:42
QC_sim_benchmark.control_qbit
control_qbit
Definition: QC_sim_benchmark.py:86
apply_crot_kernel_to_matrix_input_from_right_AVX
void apply_crot_kernel_to_matrix_input_from_right_AVX(Matrix &u3_1qbit1, Matrix &u3_1qbit2, Matrix &input, const int &target_qbit, const int &control_qbit, const int &matrix_size)
Definition: apply_large_kernel_to_input_AVX.cpp:4089
Generated on Thu Jul 9 2026 07:59:23 for Sequential Quantum Gate Decomposer by   doxygen 1.8.13