39{
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194 char * one;
195 Int Afwd, Bbufld, Bcol, Bcurcol, Bcurrow, Bii, Bimb, Bimb1, Binb,
196 Binb1, BisRow, Bjj, Bld, Bm, Bmb, Bmp, Bn, Bnb, Bnnxt, BnnxtL,
197 Bnpre, Bnq, Brow, WAfr, ctxt, kb, mycol, mydist, mydistnb,
198 myrow, nlen, npcol, nprow, offset, size, srcdist, stride,
199 stridenb, tmp;
202
203
204
206 char * Bptr = NULL, * Bbuf = NULL, * Bbufptr = NULL, * WA = NULL;
207
208
209
210
211
212
213
214 if( ( M <= 0 ) || ( N <= 0 ) ) return;
215
216
217
219
222 if(
Mupcase( AROC[0] ) ==
Mupcase( BROC[0] ) ) { Bm = M; Bn = N; }
223 else { Bm = N; Bn = M; }
224
225
226
227 PB_Cinfog2l( IB, JB, DESCB, nprow, npcol, myrow, mycol, &Bii, &Bjj,
228 &Brow, &Bcol );
229 Bimb = DESCB[
IMB_]; Binb = DESCB[
INB_]; Bmb = DESCB[
MB_]; Bnb = DESCB[
NB_];
231 Bmp =
PB_Cnumroc( Bm, 0, Bimb1, Bmb, myrow, Brow, nprow );
233 Bnq =
PB_Cnumroc( Bn, 0, Binb1, Bnb, mycol, Bcol, npcol );
235 if( ( Bmp > 0 ) && ( Bnq > 0 ) ) Bptr =
Mptr( B, Bii, Bjj, Bld, size );
236
237 if( BisRow )
238 {
239
240
241
242 if( Afwd ) { Bcurrow = Brow; }
243 else { Bcurrow =
PB_Cindxg2p( Bm-1, Bimb1, Bmb, Brow, Brow, nprow ); }
244 PB_Cdescset( Bd0, Bm, Bn, Bm, Binb1, Bmb, Bnb, Bcurrow, Bcol, ctxt, Bld );
245
246
247
248 PB_CInV(
TYPE,
NOCONJG, BROC, Bm, Bn, Bd0, Bm, A, IA, JA, DESCA, AROC,
249 &WA, WAd, &WAfr );
250
251
252
253 if( ( Brow == -1 ) || ( nprow == 1 ) )
254 {
255
256
257
258 if( Bnq > 0 )
259 TYPE->Fmmadd( &Bm, &Bnq, one, WA, &WAd[
LLD_], ALPHA, Bptr, &Bld );
260 if( WAfr ) free( WA );
261 return;
262 }
263
264 if( !(
PB_Cspan( Bm, 0, Bimb1, Bmb, Brow, nprow ) ) )
265 {
266
267
268
269 if( ( myrow == Brow ) && ( Bnq > 0 ) )
270 TYPE->Fmmadd( &Bm, &Bnq, one, WA, &WAd[
LLD_], ALPHA, Bptr, &Bld );
271 if( WAfr ) free( WA );
272 return;
273 }
274
275
276
277 if( Afwd )
278 {
279
280
281
282
283 if( ( Bmp > 0 ) && ( Bnq > 0 ) )
284 {
285
286
287
288 Bnpre =
PB_Cnpreroc( Bm, 0, Bimb1, Bmb, myrow, Brow, nprow );
289 Bnnxt =
PB_Cnnxtroc( Bm, 0, Bimb1, Bmb, myrow, Brow, nprow );
290 nlen = Bmp + Bnnxt;
291
292 if( Bnpre > 0 )
293 {
294
295
296
297
298 Bbufptr = Bbuf =
PB_Cmalloc( nlen * Bnq * size );
299 Bbufld = nlen;
300 TYPE->Cgerv2d( ctxt, nlen, Bnq, Bbuf, Bbufld,
MModSub1( myrow,
301 nprow ), mycol );
302 kb = Bmb;
303 }
304 else
305 {
306
307
308
309 Bbufptr = Bbuf = WA;
311 kb = Bimb1;
312 }
313
314
315
316 if( Bnnxt > 0 )
317 {
318
319
320
321
322 add =
TYPE->Fmmadd; shft =
TYPE->Frshft;
323 mydistnb = ( nprow -
MModSub( myrow, Brow, nprow ) - 1 );
324 stride = ( mydistnb *= Bmb ) * size;
325
326 do
327 {
328 kb =
MIN( kb, nlen );
329 add( &kb, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr, &Bld );
330 nlen -= kb;
331 offset = -kb;
332 shft( &nlen, &Bnq, &offset, Bbufptr, &Bbufld );
333 Bptr += kb*size;
334 Bbufptr += stride;
335 nlen -= mydistnb;
336 kb = Bmb;
337 } while( nlen > 0 );
338
339
340
341 TYPE->Cgesd2d( ctxt, Bnnxt, Bnq, Bbuf, Bbufld,
MModAdd1( myrow,
342 nprow ), mycol );
343 }
344 else
345 {
346
347
348
349
350 TYPE->Fmmadd( &Bmp, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr,
351 &Bld );
352 }
353
354
355
356 if( Bnpre > 0 ) free( Bbuf );
357 }
358 if( WAfr ) free( WA );
359 }
360 else
361 {
362 if( ( Bmp > 0 ) && ( Bnq > 0 ) )
363 {
364
365
366
367 Bnnxt =
PB_Cnnxtroc( Bm, 0, Bimb1, Bmb, myrow, Brow, nprow );
368 BnnxtL =
PB_Cnnxtroc( Bm, 0, Bimb1, Bmb, Bcurrow, Brow, nprow );
369 Bnnxt =
MModSub( Bnnxt, BnnxtL, Bm );
370 Bnpre = ( nlen = Bm - Bnnxt ) - Bmp;
371
372 if( Bnnxt > 0 )
373 {
374
375
376
377
378 Bbufptr = Bbuf =
PB_Cmalloc( nlen * Bnq * size );
379 Bbufld = nlen;
380 TYPE->Cgerv2d( ctxt, nlen, Bnq, Bbuf, Bbufld,
MModAdd1( myrow,
381 nprow ), mycol );
382 }
383 else
384 {
385
386
387
388 Bbufptr = Bbuf = WA;
390 }
391
392
393
394 if( Bnpre > 0 )
395 {
396
397
398
399
400 add =
TYPE->Fmmadd; shft =
TYPE->Frshft;
401 mydist =
MModSub( Bcurrow, myrow, nprow );
402 srcdist =
MModSub( Bcurrow, Brow, nprow );
403 stridenb = ( nprow - mydist - 1 ) * Bmb;
404
405 if( mydist < srcdist )
406 {
407 tmp = ( Bimb1 + ( srcdist - mydist - 1 ) * Bmb );
408 Bbufptr += tmp * size;
409 nlen -= tmp;
410 kb = Bmb;
411 }
412 else if( mydist == srcdist )
413 {
414 kb = Bimb1;
415 }
416 else
417 {
418 Bbufptr += stridenb * size;
419 nlen -= stridenb;
420 kb = Bmb;
421 }
422
423 do
424 {
425 kb =
MIN( kb, nlen );
426 add( &kb, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr, &Bld );
427 nlen -= kb;
428 offset = -kb;
429 shft( &nlen, &Bnq, &offset, Bbufptr, &Bbufld );
430 Bptr += kb*size;
431 Bbufptr += stridenb*size;
432 nlen -= stridenb;
433 kb = Bmb;
434 } while( nlen > 0 );
435
436
437
438 TYPE->Cgesd2d( ctxt, Bnpre, Bnq, Bbuf, Bbufld,
MModSub1( myrow,
439 nprow ), mycol );
440 }
441 else
442 {
443
444
445
446
447 TYPE->Fmmadd( &Bmp, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr,
448 &Bld );
449 }
450
451
452
453 if( Bnnxt > 0 ) free( Bbuf );
454 }
455 if( WAfr ) free( WA );
456 }
457 }
458 else
459 {
460
461
462
463 if( Afwd ) { Bcurcol = Bcol; }
464 else { Bcurcol =
PB_Cindxg2p( Bn-1, Binb1, Bnb, Bcol, Bcol, npcol ); }
465 PB_Cdescset( Bd0, Bm, Bn, Bimb1, Bn, Bmb, Bnb, Brow, Bcurcol, ctxt, Bld );
466
467
468
469 PB_CInV(
TYPE,
NOCONJG, BROC, Bm, Bn, Bd0, Bn, A, IA, JA, DESCA, AROC,
470 &WA, WAd, &WAfr );
471
472
473
474 if( ( Bcol == -1 ) || ( npcol == 1 ) )
475 {
476
477
478
479 if( Bmp > 0 )
480 TYPE->Fmmadd( &Bmp, &Bn, one, WA, &WAd[
LLD_], ALPHA, Bptr, &Bld );
481 if( WAfr ) free( WA );
482 return;
483 }
484
485 if( !(
PB_Cspan( Bn, 0, Binb1, Bnb, Bcol, npcol ) ) )
486 {
487
488
489
490 if( ( mycol == Bcol ) && ( Bmp > 0 ) )
491 TYPE->Fmmadd( &Bmp, &Bn, one, WA, &WAd[
LLD_], ALPHA, Bptr, &Bld );
492 if( WAfr ) free( WA );
493 return;
494 }
495
496
497
498 if( Afwd )
499 {
500
501
502
503
504
505 if( ( Bmp > 0 ) && ( Bnq > 0 ) )
506 {
507
508
509
510 Bnpre =
PB_Cnpreroc( Bn, 0, Binb1, Bnb, mycol, Bcol, npcol );
511 Bnnxt =
PB_Cnnxtroc( Bn, 0, Binb1, Bnb, mycol, Bcol, npcol );
512 nlen = Bnq + Bnnxt;
513
514 if( Bnpre > 0 )
515 {
516
517
518
519
520 Bbufptr = Bbuf =
PB_Cmalloc( Bmp * nlen * size );
521 Bbufld = Bmp;
522 TYPE->Cgerv2d( ctxt, Bmp, nlen, Bbuf, Bbufld, myrow,
524 kb = Bnb;
525 }
526 else
527 {
528
529
530
531 Bbufptr = Bbuf = WA;
533 kb = Binb1;
534 }
535
536
537
538 if( Bnnxt > 0 )
539 {
540
541
542
543
544
545 add =
TYPE->Fmmadd; shft =
TYPE->Fcshft;
546 mydistnb = ( npcol -
MModSub( mycol, Bcol, npcol ) - 1 );
547 stride = ( mydistnb *= Bnb ) * Bbufld * size;
548
549 do
550 {
551 kb =
MIN( kb, nlen );
552 add( &Bmp, &kb, one, Bbufptr, &Bbufld, ALPHA, Bptr, &Bld );
553 nlen -= kb;
554 offset = -kb;
555 shft( &Bmp, &nlen, &offset, Bbufptr, &Bbufld );
556 Bptr += kb*Bld*size;
557 Bbufptr += stride;
558 nlen -= mydistnb;
559 kb = Bnb;
560 } while( nlen > 0 );
561
562
563
564 TYPE->Cgesd2d( ctxt, Bmp, Bnnxt, Bbuf, Bbufld, myrow,
566 }
567 else
568 {
569
570
571
572
573 TYPE->Fmmadd( &Bmp, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr,
574 &Bld );
575 }
576
577
578
579 if( Bnpre > 0 ) free( Bbuf );
580 }
581 if( WAfr ) free( WA );
582 }
583 else
584 {
585 if( ( Bmp > 0 ) && ( Bnq > 0 ) )
586 {
587
588
589
590 Bnnxt =
PB_Cnnxtroc( Bn, 0, Binb1, Bnb, mycol, Bcol, npcol );
591 BnnxtL =
PB_Cnnxtroc( Bn, 0, Binb1, Bnb, Bcurcol, Bcol, npcol );
592 Bnnxt =
MModSub( Bnnxt, BnnxtL, Bn );
593 Bnpre = ( nlen = Bn - Bnnxt ) - Bnq;
594
595 if( Bnnxt > 0 )
596 {
597
598
599
600
601 Bbufptr = Bbuf =
PB_Cmalloc( Bmp * nlen * size );
602 Bbufld = Bmp;
603 TYPE->Cgerv2d( ctxt, Bmp, nlen, Bbuf, Bbufld, myrow,
605 }
606 else
607 {
608
609
610
611 Bbufptr = Bbuf = WA;
613 }
614
615
616
617 if( Bnpre > 0 )
618 {
619
620
621
622
623
624 add =
TYPE->Fmmadd; shft =
TYPE->Fcshft;
625 mydist =
MModSub( Bcurcol, mycol, npcol );
626 srcdist =
MModSub( Bcurcol, Bcol, npcol );
627 stridenb = ( npcol - mydist - 1 ) * Bnb;
628
629 if( mydist < srcdist )
630 {
631 tmp = ( Binb1 + ( srcdist - mydist - 1 ) * Bnb );
632 Bbufptr += tmp * Bbufld * size;
633 nlen -= tmp;
634 kb = Bnb;
635 }
636 else if( mydist == srcdist )
637 {
638 kb = Binb1;
639 }
640 else
641 {
642 Bbufptr += stridenb * Bbufld * size;
643 nlen -= stridenb;
644 kb = Bnb;
645 }
646
647 do
648 {
649 kb =
MIN( kb, nlen );
650 add( &Bmp, &kb, one, Bbufptr, &Bbufld, ALPHA, Bptr, &Bld );
651 nlen -= kb;
652 offset = -kb;
653 shft( &Bmp, &nlen, &offset, Bbufptr, &Bbufld );
654 Bptr += kb * Bld * size;
655 Bbufptr += stridenb * Bbufld * size;
656 nlen -= stridenb;
657 kb = Bnb;
658 } while( nlen > 0 );
659
660
661
662 TYPE->Cgesd2d( ctxt, Bmp, Bnpre, Bbuf, Bbufld, myrow,
664 }
665 else
666 {
667
668
669
670
671 TYPE->Fmmadd( &Bmp, &Bnq, one, Bbufptr, &Bbufld, ALPHA, Bptr,
672 &Bld );
673 }
674
675
676
677
678 if( Bnnxt > 0 ) free( Bbuf );
679 }
680 if( WAfr ) free( WA );
681 }
682 }
683
684
685
686}