summaryrefslogtreecommitdiffstats
path: root/crypto/Kconfig
blob: 8aaf298a80e165f3fb5f83b1e00b8636cf9d08a3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
#
# Generic algorithms support
#
config XOR_BLOCKS
	tristate

#
# async_tx api: hardware offloaded memory transfer/transform support
#
source "crypto/async_tx/Kconfig"

#
# Cryptographic API Configuration
#
menuconfig CRYPTO
	tristate "Cryptographic API"
	help
	  This option provides the core Cryptographic API.

if CRYPTO

comment "Crypto core or helper"

config CRYPTO_FIPS
	bool "FIPS 200 compliance"
	depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS
	depends on MODULE_SIG
	help
	  This options enables the fips boot option which is
	  required if you want to system to operate in a FIPS 200
	  certification.  You should say no unless you know what
	  this is.

config CRYPTO_ALGAPI
	tristate
	select CRYPTO_ALGAPI2
	help
	  This option provides the API for cryptographic algorithms.

config CRYPTO_ALGAPI2
	tristate

config CRYPTO_AEAD
	tristate
	select CRYPTO_AEAD2
	select CRYPTO_ALGAPI

config CRYPTO_AEAD2
	tristate
	select CRYPTO_ALGAPI2

config CRYPTO_BLKCIPHER
	tristate
	select CRYPTO_BLKCIPHER2
	select CRYPTO_ALGAPI

config CRYPTO_BLKCIPHER2
	tristate
	select CRYPTO_ALGAPI2
	select CRYPTO_RNG2
	select CRYPTO_WORKQUEUE

config CRYPTO_HASH
	tristate
	select CRYPTO_HASH2
	select CRYPTO_ALGAPI

config CRYPTO_HASH2
	tristate
	select CRYPTO_ALGAPI2

config CRYPTO_RNG
	tristate
	select CRYPTO_RNG2
	select CRYPTO_ALGAPI

config CRYPTO_RNG2
	tristate
	select CRYPTO_ALGAPI2

config CRYPTO_PCOMP
	tristate
	select CRYPTO_PCOMP2
	select CRYPTO_ALGAPI

config CRYPTO_PCOMP2
	tristate
	select CRYPTO_ALGAPI2

config CRYPTO_MANAGER
	tristate "Cryptographic algorithm manager"
	select CRYPTO_MANAGER2
	help
	  Create default cryptographic template instantiations such as
	  cbc(aes).

config CRYPTO_MANAGER2
	def_tristate CRYPTO_MANAGER || (CRYPTO_MANAGER!=n && CRYPTO_ALGAPI=y)
	select CRYPTO_AEAD2
	select CRYPTO_HASH2
	select CRYPTO_BLKCIPHER2
	select CRYPTO_PCOMP2

config CRYPTO_USER
	tristate "Userspace cryptographic algorithm configuration"
	depends on NET
	select CRYPTO_MANAGER
	help
	  Userspace configuration for cryptographic instantiations such as
	  cbc(aes).

config CRYPTO_MANAGER_DISABLE_TESTS
	bool "Disable run-time self tests"
	default y
	depends on CRYPTO_MANAGER2
	help
	  Disable run-time self tests that normally take place at
	  algorithm registration.

config CRYPTO_GF128MUL
	tristate "GF(2^128) multiplication functions"
	help
	  Efficient table driven implementation of multiplications in the
	  field GF(2^128).  This is needed by some cypher modes. This
	  option will be selected automatically if you select such a
	  cipher mode.  Only select this option by hand if you expect to load
	  an external module that requires these functions.

config CRYPTO_NULL
	tristate "Null algorithms"
	select CRYPTO_ALGAPI
	select CRYPTO_BLKCIPHER
	select CRYPTO_HASH
	help
	  These are 'Null' algorithms, used by IPsec, which do nothing.

config CRYPTO_PCRYPT
	tristate "Parallel crypto engine"
	depends on SMP
	select PADATA
	select CRYPTO_MANAGER
	select CRYPTO_AEAD
	help
	  This converts an arbitrary crypto algorithm into a parallel
	  algorithm that executes in kernel threads.

config CRYPTO_WORKQUEUE
       tristate

config CRYPTO_CRYPTD
	tristate "Software async crypto daemon"
	select CRYPTO_BLKCIPHER
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	select CRYPTO_WORKQUEUE
	help
	  This is a generic software asynchronous crypto daemon that
	  converts an arbitrary synchronous software crypto algorithm
	  into an asynchronous algorithm that executes in a kernel thread.

config CRYPTO_MCRYPTD
	tristate "Software async multi-buffer crypto daemon"
	select CRYPTO_BLKCIPHER
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	select CRYPTO_WORKQUEUE
	help
	  This is a generic software asynchronous crypto daemon that
	  provides the kernel thread to assist multi-buffer crypto
	  algorithms for submitting jobs and flushing jobs in multi-buffer
	  crypto algorithms.  Multi-buffer crypto algorithms are executed
	  in the context of this kernel thread and drivers can post
	  their crypto request asynchronously to be processed by this daemon.

config CRYPTO_AUTHENC
	tristate "Authenc support"
	select CRYPTO_AEAD
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	select CRYPTO_HASH
	help
	  Authenc: Combined mode wrapper for IPsec.
	  This is required for IPSec.

config CRYPTO_TEST
	tristate "Testing module"
	depends on m
	select CRYPTO_MANAGER
	help
	  Quick & dirty crypto test module.

config CRYPTO_ABLK_HELPER
	tristate
	select CRYPTO_CRYPTD

config CRYPTO_GLUE_HELPER_X86
	tristate
	depends on X86
	select CRYPTO_ALGAPI

comment "Authenticated Encryption with Associated Data"

config CRYPTO_CCM
	tristate "CCM support"
	select CRYPTO_CTR
	select CRYPTO_AEAD
	help
	  Support for Counter with CBC MAC. Required for IPsec.

config CRYPTO_GCM
	tristate "GCM/GMAC support"
	select CRYPTO_CTR
	select CRYPTO_AEAD
	select CRYPTO_GHASH
	select CRYPTO_NULL
	help
	  Support for Galois/Counter Mode (GCM) and Galois Message
	  Authentication Code (GMAC). Required for IPSec.

config CRYPTO_SEQIV
	tristate "Sequence Number IV Generator"
	select CRYPTO_AEAD
	select CRYPTO_BLKCIPHER
	select CRYPTO_RNG
	help
	  This IV generator generates an IV based on a sequence number by
	  xoring it with a salt.  This algorithm is mainly useful for CTR

comment "Block modes"

config CRYPTO_CBC
	tristate "CBC support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	help
	  CBC: Cipher Block Chaining mode
	  This block cipher algorithm is required for IPSec.

config CRYPTO_CTR
	tristate "CTR support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_SEQIV
	select CRYPTO_MANAGER
	help
	  CTR: Counter mode
	  This block cipher algorithm is required for IPSec.

config CRYPTO_CTS
	tristate "CTS support"
	select CRYPTO_BLKCIPHER
	help
	  CTS: Cipher Text Stealing
	  This is the Cipher Text Stealing mode as described by
	  Section 8 of rfc2040 and referenced by rfc3962.
	  (rfc3962 includes errata information in its Appendix A)
	  This mode is required for Kerberos gss mechanism support
	  for AES encryption.

config CRYPTO_ECB
	tristate "ECB support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	help
	  ECB: Electronic CodeBook mode
	  This is the simplest block cipher algorithm.  It simply encrypts
	  the input block by block.

config CRYPTO_LRW
	tristate "LRW support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	select CRYPTO_GF128MUL
	help
	  LRW: Liskov Rivest Wagner, a tweakable, non malleable, non movable
	  narrow block cipher mode for dm-crypt.  Use it with cipher
	  specification string aes-lrw-benbi, the key must be 256, 320 or 384.
	  The first 128, 192 or 256 bits in the key are used for AES and the
	  rest is used to tie each cipher block to its logical position.

config CRYPTO_PCBC
	tristate "PCBC support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	help
	  PCBC: Propagating Cipher Block Chaining mode
	  This block cipher algorithm is required for RxRPC.

config CRYPTO_XTS
	tristate "XTS support"
	select CRYPTO_BLKCIPHER
	select CRYPTO_MANAGER
	select CRYPTO_GF128MUL
	help
	  XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain,
	  key size 256, 384 or 512 bits. This implementation currently
	  can't handle a sectorsize which is not a multiple of 16 bytes.

comment "Hash modes"

config CRYPTO_CMAC
	tristate "CMAC support"
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	help
	  Cipher-based Message Authentication Code (CMAC) specified by
	  The National Institute of Standards and Technology (NIST).

	  https://tools.ietf.org/html/rfc4493
	  http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf

config CRYPTO_HMAC
	tristate "HMAC support"
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	help
	  HMAC: Keyed-Hashing for Message Authentication (RFC2104).
	  This is required for IPSec.

config CRYPTO_XCBC
	tristate "XCBC support"
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	help
	  XCBC: Keyed-Hashing with encryption algorithm
		http://www.ietf.org/rfc/rfc3566.txt
		http://csrc.nist.gov/encryption/modes/proposedmodes/
		 xcbc-mac/xcbc-mac-spec.pdf

config CRYPTO_VMAC
	tristate "VMAC support"
	select CRYPTO_HASH
	select CRYPTO_MANAGER
	help
	  VMAC is a message authentication algorithm designed for
	  very high speed on 64-bit architectures.

	  See also:
	  <http://fastcrypto.org/vmac>

comment "Digest"

config CRYPTO_CRC32C
	tristate "CRC32c CRC algorithm"
	select CRYPTO_HASH
	select CRC32
	help
	  Castagnoli, et al Cyclic Redundancy-Check Algorithm.  Used
	  by iSCSI for header and data digests and by others.
	  See Castagnoli93.  Module will be crc32c.

config CRYPTO_CRC32C_INTEL
	tristate "CRC32c INTEL hardware acceleration"
	depends on X86
	select CRYPTO_HASH
	help
	  In Intel processor with SSE4.2 supported, the processor will
	  support CRC32C implementation using hardware accelerated CRC32
	  instruction. This option will create 'crc32c-intel' module,
	  which will enable any routine to use the CRC32 instruction to
	  gain performance compared with software implementation.
	  Module will be crc32c-intel.

config CRYPTO_CRC32C_SPARC64
	tristate "CRC32c CRC algorithm (SPARC64)"
	depends on SPARC64
	select CRYPTO_HASH
	select CRC32
	help
	  CRC32c CRC algorithm implemented using sparc64 crypto instructions,
	  when available.

config CRYPTO_CRC32
	tristate "CRC32 CRC algorithm"
	select CRYPTO_HASH
	select CRC32
	help
	  CRC-32-IEEE 802.3 cyclic redundancy-check algorithm.
	  Shash crypto api wrappers to crc32_le function.

config CRYPTO_CRC32_PCLMUL
	tristate "CRC32 PCLMULQDQ hardware acceleration"
	depends on X86
	select CRYPTO_HASH
	select CRC32
	help
	  From Intel Westmere and AMD Bulldozer processor with SSE4.2
	  and PCLMULQDQ supported, the processor will support
	  CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ
	  instruction. This option will create 'crc32-plcmul' module,
	  which will enable any routine to use the CRC-32-IEEE 802.3 checksum
	  and gain better performance as compared with the table implementation.

config CRYPTO_CRCT10DIF
	tristate "CRCT10DIF algorithm"
	select CRYPTO_HASH
	help
	  CRC T10 Data Integrity Field computation is being cast as
	  a crypto transform.  This allows for faster crc t10 diff
	  transforms to be used if they are available.

config CRYPTO_CRCT10DIF_PCLMUL
	tristate "CRCT10DIF PCLMULQDQ hardware acceleration"
	depends on X86 && 64BIT && CRC_T10DIF
	select CRYPTO_HASH
	help
	  For x86_64 processors with SSE4.2 and PCLMULQDQ supported,
	  CRC T10 DIF PCLMULQDQ computation can be hardware
	  accelerated PCLMULQDQ instruction. This option will create
	  'crct10dif-plcmul' module, which is faster when computing the
	  crct10dif checksum as compared with the generic table implementation.

config CRYPTO_GHASH
	tristate "GHASH digest algorithm"
	select CRYPTO_GF128MUL
	help
	  GHASH is message digest algorithm for GCM (Galois/Counter Mode).

config CRYPTO_MD4
	tristate "MD4 digest algorithm"
	select CRYPTO_HASH
	help
	  MD4 message digest algorithm (RFC1320).

config CRYPTO_MD5
	tristate "MD5 digest algorithm"
	select CRYPTO_HASH
	help
	  MD5 message digest algorithm (RFC1321).

config CRYPTO_MD5_OCTEON
	tristate "MD5 digest algorithm (OCTEON)"
	depends on CPU_CAVIUM_OCTEON
	select CRYPTO_MD5
	select CRYPTO_HASH
	help
	  MD5 message digest algorithm (RFC1321) implemented
	  using OCTEON crypto instructions, when available.

config CRYPTO_MD5_PPC
	tristate "MD5 digest algorithm (PPC)"
	depends on PPC
	select CRYPTO_HASH
	help
	  MD5 message digest algorithm (RFC1321) implemented
	  in PPC assembler.

config CRYPTO_MD5_SPARC64
	tristate "MD5 digest algorithm (SPARC64)"
	depends on SPARC64
	select CRYPTO_MD5
	select CRYPTO_HASH
	help
	  MD5 message digest algorithm (RFC1321) implemented
	  using sparc64 crypto instructions, when available.

config CRYPTO_MICHAEL_MIC
	tristate "Michael MIC keyed digest algorithm"
	select CRYPTO_HASH
	help
	  Michael MIC is used for message integrity protection in TKIP
	  (IEEE 802.11i). This algorithm is required for TKIP, but it
	  should not be used for other purposes because of the weakness
	  of the algorithm.

config CRYPTO_RMD128
	tristate "RIPEMD-128 digest algorithm"
	select CRYPTO_HASH
	help
	  RIPEMD-128 (ISO/IEC 10118-3:2004).

	  RIPEMD-128 is a 128-bit cryptographic hash function. It should only
	  be used as a secure replacement for RIPEMD. For other use cases,
	  RIPEMD-160 should be used.

	  Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel.
	  See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html>

config CRYPTO_RMD160
	tristate "RIPEMD-160 digest algorithm"
	select CRYPTO_HASH
	help
	  RIPEMD-160 (ISO/IEC 10118-3:2004).

	  RIPEMD-160 is a 160-bit cryptographic hash function. It is intended
	  to be used as a secure replacement for the 128-bit hash functions
	  MD4, MD5 and it's predecessor RIPEMD
	  (not to be confused with RIPEMD-128).

	  It's speed is comparable to SHA1 and there are no known attacks
	  against RIPEMD-160.

	  Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel.
	  See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html>

config CRYPTO_RMD256
	tristate "RIPEMD-256 digest algorithm"
	select CRYPTO_HASH
	help
	  RIPEMD-256 is an optional extension of RIPEMD-128 with a
	  256 bit hash. It is intended for applications that require
	  longer hash-results, without needing a larger security level
	  (than RIPEMD-128).

	  Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel.
	  See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html>

config CRYPTO_RMD320
	tristate "RIPEMD-320 digest algorithm"
	select CRYPTO_HASH
	help
	  RIPEMD-320 is an optional extension of RIPEMD-160 with a
	  320 bit hash. It is intended for applications that require
	  longer hash-results, without needing a larger security level
	  (than RIPEMD-160).

	  Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel.
	  See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html>

config CRYPTO_SHA1
	tristate "SHA1 digest algorithm"
	select CRYPTO_HASH
	help
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).

config CRYPTO_SHA1_SSSE3
	tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2)"
	depends on X86 && 64BIT
	select CRYPTO_SHA1
	select CRYPTO_HASH
	help
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
	  using Supplemental SSE3 (SSSE3) instructions or Advanced Vector
	  Extensions (AVX/AVX2), when available.

config CRYPTO_SHA256_SSSE3
	tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2)"
	depends on X86 && 64BIT
	select CRYPTO_SHA256
	select CRYPTO_HASH
	help
	  SHA-256 secure hash standard (DFIPS 180-2) implemented
	  using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector
	  Extensions version 1 (AVX1), or Advanced Vector Extensions
	  version 2 (AVX2) instructions, when available.

config CRYPTO_SHA512_SSSE3
	tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)"
	depends on X86 && 64BIT
	select CRYPTO_SHA512
	select CRYPTO_HASH
	help
	  SHA-512 secure hash standard (DFIPS 180-2) implemented
	  using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector
	  Extensions version 1 (AVX1), or Advanced Vector Extensions
	  version 2 (AVX2) instructions, when available.

config CRYPTO_SHA1_OCTEON
	tristate "SHA1 digest algorithm (OCTEON)"
	depends on CPU_CAVIUM_OCTEON
	select CRYPTO_SHA1
	select CRYPTO_HASH
	help
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
	  using OCTEON crypto instructions, when available.

config CRYPTO_SHA1_SPARC64
	tristate "SHA1 digest algorithm (SPARC64)"
	depends on SPARC64
	select CRYPTO_SHA1
	select CRYPTO_HASH
	help
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
	  using sparc64 crypto instructions, when available.

config CRYPTO_SHA1_PPC
	tristate "SHA1 digest algorithm (powerpc)"
	depends on PPC
	help
	  This is the powerpc hardware accelerated implementation of the
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).

config CRYPTO_SHA1_PPC_SPE
	tristate "SHA1 digest algorithm (PPC SPE)"
	depends on PPC && SPE
	help
	  SHA-1 secure hash standard (DFIPS 180-4) implemented
	  using powerpc SPE SIMD instruction set.

config CRYPTO_SHA1_MB
	tristate "SHA1 digest algorithm (x86_64 Multi-Buffer, Experimental)"
	depends on X86 && 64BIT
	select CRYPTO_SHA1
	select CRYPTO_HASH
	select CRYPTO_MCRYPTD
	help
	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
	  using multi-buffer technique.  This algorithm computes on
	  multiple data lanes concurrently with SIMD instructions for
	  better throughput.  It should not be enabled by default but
	  used when there is significant amount of work to keep the keep
	  the data lanes filled to get performance benefit.  If the data
	  lanes remain unfilled, a flush operation will be initiated to
	  process the crypto jobs, adding a slight latency.

config CRYPTO_SHA256
	tristate "SHA224 and SHA256 digest algorithm"
	select CRYPTO_HASH
	help
	  SHA256 secure hash standard (DFIPS 180-2).

	  This version of SHA implements a 256 bit hash with 128 bits of
	  security against collision attacks.

	  This code also includes SHA-224, a 224 bit hash with 112 bits
	  of security against collision attacks.

config CRYPTO_SHA256_PPC_SPE
	tristate "SHA224 and SHA256 digest algorithm (PPC SPE)"
	depends on PPC && SPE
	select CRYPTO_SHA256
	select CRYPTO_HASH
	help
	  SHA224 and SHA256 secure hash standard (DFIPS 180-2)
	  implemented using powerpc SPE SIMD instruction set.

config CRYPTO_SHA256_OCTEON
	tristate "SHA224 and SHA256 digest algorithm (OCTEON)"
	depends on CPU_CAVIUM_OCTEON
	select CRYPTO_SHA256
	select CRYPTO_HASH
	help
	  SHA-256 secure hash standard (DFIPS 180-2) implemented
	  using OCTEON crypto instructions, when available.

config CRYPTO_SHA256_SPARC64
	tristate "SHA224 and SHA256 digest algorithm (SPARC64)"
	depends on SPARC64
	select CRYPTO_SHA256
	select CRYPTO_HASH
	help
	  SHA-256 secure hash standard (DFIPS 180-2) implemented
	  using sparc64 crypto instructions, when available.

config CRYPTO_SHA512
	tristate "SHA384 and SHA512 digest algorithms"
	select CRYPTO_HASH
	help
	  SHA512 secure hash standard (DFIPS 180-2).

	  This version of SHA implements a 512 bit hash with 256 bits of
	  security against collision attacks.

	  This code also includes SHA-384, a 384 bit hash with 192 bits
	  of security against collision attacks.

config CRYPTO_SHA512_OCTEON
	tristate "SHA384 and SHA512 digest algorithms (OCTEON)"
	depends on CPU_CAVIUM_OCTEON
	select CRYPTO_SHA512
	select CRYPTO_HASH
	help
	  SHA-512 secure hash standard (DFIPS 180-2) implemented
	  using OCTEON crypto instructions, when available.

config CRYPTO_SHA512_SPARC64
	tristate "SHA384 and SHA512 digest algorithm (SPARC64)"
	depends on SPARC64
	select CRYPTO_SHA512
	select CRYPTO_HASH
	help
	  SHA-512 secure hash standard (DFIPS 180-2) implemented
	  using sparc64 crypto instructions, when available.

config CRYPTO_TGR192
	tristate "Tiger digest algorithms"
	select CRYPTO_HASH
	help
	  Tiger hash algorithm 192, 160 and 128-bit hashes

	  Tiger is a hash function optimized for 64-bit processors while
	  still having decent performance on 32-bit processors.
	  Tiger was developed by Ross Anderson and Eli Biham.

	  See also:
	  <http://www.cs.technion.ac.il/~biham/Reports/Tiger/>.

config CRYPTO_WP512
	tristate "Whirlpool digest algorithms"
	select CRYPTO_HASH
	help
	  Whirlpool hash algorithm 512, 384 and 256-bit hashes

	  Whirlpool-512 is part of the NESSIE cryptographic primitives.
	  Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard

	  See also:
	  <http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html>

config CRYPTO_GHASH_CLMUL_NI_INTEL
	tristate "GHASH digest algorithm (CLMUL-NI accelerated)"
	depends on X86 && 64BIT
	select CRYPTO_CRYPTD
	help
	  GHASH is message digest algorithm for GCM (Galois/Counter Mode).
	  The implementation is accelerated by CLMUL-NI of Intel.

comment "Ciphers"

config CRYPTO_AES
	tristate "AES cipher algorithms"
	select CRYPTO_ALGAPI
	help
	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
	  algorithm.

	  Rijndael appears to be consistently a very good performer in
	  both hardware and software across a wide range of computing
	  environments regardless of its use in feedback or non-feedback
	  modes. Its key setup time is excellent, and its key agility is
	  good. Rijndael's very low memory requirements make it very well
	  suited for restricted-space environments, in which it also
	  demonstrates excellent performance. Rijndael's operations are
	  among the easiest to defend against power and timing attacks.

	  The AES specifies three key sizes: 128, 192 and 256 bits

	  See <http://csrc.nist.gov/CryptoToolkit/aes/> for more information.

config CRYPTO_AES_586
	tristate "AES cipher algorithms (i586)"
	depends on (X86 || UML_X86) && !64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_AES
	help
	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
	  algorithm.

	  Rijndael appears to be consistently a very good performer in
	  both hardware and software across a wide range of computing
	  environments regardless of its use in feedback or non-feedback
	  modes. Its key setup time is excellent, and its key agility is
	  good. Rijndael's very low memory requirements make it very well
	  suited for restricted-space environments, in which it also
	  demonstrates excellent performance. Rijndael's operations are
	  among the easiest to defend against power and timing attacks.

	  The AES specifies three key sizes: 128, 192 and 256 bits

	  See <http://csrc.nist.gov/encryption/aes/> for more information.

config CRYPTO_AES_X86_64
	tristate "AES cipher algorithms (x86_64)"
	depends on (X86 || UML_X86) && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_AES
	help
	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
	  algorithm.

	  Rijndael appears to be consistently a very good performer in
	  both hardware and software across a wide range of computing
	  environments regardless of its use in feedback or non-feedback
	  modes. Its key setup time is excellent, and its key agility is
	  good. Rijndael's very low memory requirements make it very well
	  suited for restricted-space environments, in which it also
	  demonstrates excellent performance. Rijndael's operations are
	  among the easiest to defend against power and timing attacks.

	  The AES specifies three key sizes: 128, 192 and 256 bits

	  See <http://csrc.nist.gov/encryption/aes/> for more information.

config CRYPTO_AES_NI_INTEL
	tristate "AES cipher algorithms (AES-NI)"
	depends on X86
	select CRYPTO_AES_X86_64 if 64BIT
	select CRYPTO_AES_586 if !64BIT
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_ALGAPI
	select CRYPTO_GLUE_HELPER_X86 if 64BIT
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Use Intel AES-NI instructions for AES algorithm.

	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
	  algorithm.

	  Rijndael appears to be consistently a very good performer in
	  both hardware and software across a wide range of computing
	  environments regardless of its use in feedback or non-feedback
	  modes. Its key setup time is excellent, and its key agility is
	  good. Rijndael's very low memory requirements make it very well
	  suited for restricted-space environments, in which it also
	  demonstrates excellent performance. Rijndael's operations are
	  among the easiest to defend against power and timing attacks.

	  The AES specifies three key sizes: 128, 192 and 256 bits

	  See <http://csrc.nist.gov/encryption/aes/> for more information.

	  In addition to AES cipher algorithm support, the acceleration
	  for some popular block cipher mode is supported too, including
	  ECB, CBC, LRW, PCBC, XTS. The 64 bit version has additional
	  acceleration for CTR.

config CRYPTO_AES_SPARC64
	tristate "AES cipher algorithms (SPARC64)"
	depends on SPARC64
	select CRYPTO_CRYPTD
	select CRYPTO_ALGAPI
	help
	  Use SPARC64 crypto opcodes for AES algorithm.

	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
	  algorithm.

	  Rijndael appears to be consistently a very good performer in
	  both hardware and software across a wide range of computing
	  environments regardless of its use in feedback or non-feedback
	  modes. Its key setup time is excellent, and its key agility is
	  good. Rijndael's very low memory requirements make it very well
	  suited for restricted-space environments, in which it also
	  demonstrates excellent performance. Rijndael's operations are
	  among the easiest to defend against power and timing attacks.

	  The AES specifies three key sizes: 128, 192 and 256 bits

	  See <http://csrc.nist.gov/encryption/aes/> for more information.

	  In addition to AES cipher algorithm support, the acceleration
	  for some popular block cipher mode is supported too, including
	  ECB and CBC.

config CRYPTO_AES_PPC_SPE
	tristate "AES cipher algorithms (PPC SPE)"
	depends on PPC && SPE
	help
	  AES cipher algorithms (FIPS-197). Additionally the acceleration
	  for popular block cipher modes ECB, CBC, CTR and XTS is supported.
	  This module should only be used for low power (router) devices
	  without hardware AES acceleration (e.g. caam crypto). It reduces the
	  size of the AES tables from 16KB to 8KB + 256 bytes and mitigates
	  timining attacks. Nevertheless it might be not as secure as other
	  architecture specific assembler implementations that work on 1KB
	  tables or 256 bytes S-boxes.

config CRYPTO_ANUBIS
	tristate "Anubis cipher algorithm"
	select CRYPTO_ALGAPI
	help
	  Anubis cipher algorithm.

	  Anubis is a variable key length cipher which can use keys from
	  128 bits to 320 bits in length.  It was evaluated as a entrant
	  in the NESSIE competition.

	  See also:
	  <https://www.cosic.esat.kuleuven.be/nessie/reports/>
	  <http://www.larc.usp.br/~pbarreto/AnubisPage.html>

config CRYPTO_ARC4
	tristate "ARC4 cipher algorithm"
	select CRYPTO_BLKCIPHER
	help
	  ARC4 cipher algorithm.

	  ARC4 is a stream cipher using keys ranging from 8 bits to 2048
	  bits in length.  This algorithm is required for driver-based
	  WEP, but it should not be for other purposes because of the
	  weakness of the algorithm.

config CRYPTO_BLOWFISH
	tristate "Blowfish cipher algorithm"
	select CRYPTO_ALGAPI
	select CRYPTO_BLOWFISH_COMMON
	help
	  Blowfish cipher algorithm, by Bruce Schneier.

	  This is a variable key length cipher which can use keys from 32
	  bits to 448 bits in length.  It's fast, simple and specifically
	  designed for use on "large microprocessors".

	  See also:
	  <http://www.schneier.com/blowfish.html>

config CRYPTO_BLOWFISH_COMMON
	tristate
	help
	  Common parts of the Blowfish cipher algorithm shared by the
	  generic c and the assembler implementations.

	  See also:
	  <http://www.schneier.com/blowfish.html>

config CRYPTO_BLOWFISH_X86_64
	tristate "Blowfish cipher algorithm (x86_64)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_BLOWFISH_COMMON
	help
	  Blowfish cipher algorithm (x86_64), by Bruce Schneier.

	  This is a variable key length cipher which can use keys from 32
	  bits to 448 bits in length.  It's fast, simple and specifically
	  designed for use on "large microprocessors".

	  See also:
	  <http://www.schneier.com/blowfish.html>

config CRYPTO_CAMELLIA
	tristate "Camellia cipher algorithms"
	depends on CRYPTO
	select CRYPTO_ALGAPI
	help
	  Camellia cipher algorithms module.

	  Camellia is a symmetric key block cipher developed jointly
	  at NTT and Mitsubishi Electric Corporation.

	  The Camellia specifies three key sizes: 128, 192 and 256 bits.

	  See also:
	  <https://info.isl.ntt.co.jp/crypt/eng/camellia/index_s.html>

config CRYPTO_CAMELLIA_X86_64
	tristate "Camellia cipher algorithm (x86_64)"
	depends on X86 && 64BIT
	depends on CRYPTO
	select CRYPTO_ALGAPI
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Camellia cipher algorithm module (x86_64).

	  Camellia is a symmetric key block cipher developed jointly
	  at NTT and Mitsubishi Electric Corporation.

	  The Camellia specifies three key sizes: 128, 192 and 256 bits.

	  See also:
	  <https://info.isl.ntt.co.jp/crypt/eng/camellia/index_s.html>

config CRYPTO_CAMELLIA_AESNI_AVX_X86_64
	tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)"
	depends on X86 && 64BIT
	depends on CRYPTO
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_CAMELLIA_X86_64
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Camellia cipher algorithm module (x86_64/AES-NI/AVX).

	  Camellia is a symmetric key block cipher developed jointly
	  at NTT and Mitsubishi Electric Corporation.

	  The Camellia specifies three key sizes: 128, 192 and 256 bits.

	  See also:
	  <https://info.isl.ntt.co.jp/crypt/eng/camellia/index_s.html>

config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64
	tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)"
	depends on X86 && 64BIT
	depends on CRYPTO
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_CAMELLIA_X86_64
	select CRYPTO_CAMELLIA_AESNI_AVX_X86_64
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Camellia cipher algorithm module (x86_64/AES-NI/AVX2).

	  Camellia is a symmetric key block cipher developed jointly
	  at NTT and Mitsubishi Electric Corporation.

	  The Camellia specifies three key sizes: 128, 192 and 256 bits.

	  See also:
	  <https://info.isl.ntt.co.jp/crypt/eng/camellia/index_s.html>

config CRYPTO_CAMELLIA_SPARC64
	tristate "Camellia cipher algorithm (SPARC64)"
	depends on SPARC64
	depends on CRYPTO
	select CRYPTO_ALGAPI
	help
	  Camellia cipher algorithm module (SPARC64).

	  Camellia is a symmetric key block cipher developed jointly
	  at NTT and Mitsubishi Electric Corporation.

	  The Camellia specifies three key sizes: 128, 192 and 256 bits.

	  See also:
	  <https://info.isl.ntt.co.jp/crypt/eng/camellia/index_s.html>

config CRYPTO_CAST_COMMON
	tristate
	help
	  Common parts of the CAST cipher algorithms shared by the
	  generic c and the assembler implementations.

config CRYPTO_CAST5
	tristate "CAST5 (CAST-128) cipher algorithm"
	select CRYPTO_ALGAPI
	select CRYPTO_CAST_COMMON
	help
	  The CAST5 encryption algorithm (synonymous with CAST-128) is
	  described in RFC2144.

config CRYPTO_CAST5_AVX_X86_64
	tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_CAST_COMMON
	select CRYPTO_CAST5
	help
	  The CAST5 encryption algorithm (synonymous with CAST-128) is
	  described in RFC2144.

	  This module provides the Cast5 cipher algorithm that processes
	  sixteen blocks parallel using the AVX instruction set.

config CRYPTO_CAST6
	tristate "CAST6 (CAST-256) cipher algorithm"
	select CRYPTO_ALGAPI
	select CRYPTO_CAST_COMMON
	help
	  The CAST6 encryption algorithm (synonymous with CAST-256) is
	  described in RFC2612.

config CRYPTO_CAST6_AVX_X86_64
	tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_CAST_COMMON
	select CRYPTO_CAST6
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  The CAST6 encryption algorithm (synonymous with CAST-256) is
	  described in RFC2612.

	  This module provides the Cast6 cipher algorithm that processes
	  eight blocks parallel using the AVX instruction set.

config CRYPTO_DES
	tristate "DES and Triple DES EDE cipher algorithms"
	select CRYPTO_ALGAPI
	help
	  DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).

config CRYPTO_DES_SPARC64
	tristate "DES and Triple DES EDE cipher algorithms (SPARC64)"
	depends on SPARC64
	select CRYPTO_ALGAPI
	select CRYPTO_DES
	help
	  DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3),
	  optimized using SPARC64 crypto opcodes.

config CRYPTO_DES3_EDE_X86_64
	tristate "Triple DES EDE cipher algorithm (x86-64)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_DES
	help
	  Triple DES EDE (FIPS 46-3) algorithm.

	  This module provides implementation of the Triple DES EDE cipher
	  algorithm that is optimized for x86-64 processors. Two versions of
	  algorithm are provided; regular processing one input block and
	  one that processes three blocks parallel.

config CRYPTO_FCRYPT
	tristate "FCrypt cipher algorithm"
	select CRYPTO_ALGAPI
	select CRYPTO_BLKCIPHER
	help
	  FCrypt algorithm used by RxRPC.

config CRYPTO_KHAZAD
	tristate "Khazad cipher algorithm"
	select CRYPTO_ALGAPI
	help
	  Khazad cipher algorithm.

	  Khazad was a finalist in the initial NESSIE competition.  It is
	  an algorithm optimized for 64-bit processors with good performance
	  on 32-bit processors.  Khazad uses an 128 bit key size.

	  See also:
	  <http://www.larc.usp.br/~pbarreto/KhazadPage.html>

config CRYPTO_SALSA20
	tristate "Salsa20 stream cipher algorithm"
	select CRYPTO_BLKCIPHER
	help
	  Salsa20 stream cipher algorithm.

	  Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT
	  Stream Cipher Project. See <http://www.ecrypt.eu.org/stream/>

	  The Salsa20 stream cipher algorithm is designed by Daniel J.
	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>

config CRYPTO_SALSA20_586
	tristate "Salsa20 stream cipher algorithm (i586)"
	depends on (X86 || UML_X86) && !64BIT
	select CRYPTO_BLKCIPHER
	help
	  Salsa20 stream cipher algorithm.

	  Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT
	  Stream Cipher Project. See <http://www.ecrypt.eu.org/stream/>

	  The Salsa20 stream cipher algorithm is designed by Daniel J.
	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>

config CRYPTO_SALSA20_X86_64
	tristate "Salsa20 stream cipher algorithm (x86_64)"
	depends on (X86 || UML_X86) && 64BIT
	select CRYPTO_BLKCIPHER
	help
	  Salsa20 stream cipher algorithm.

	  Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT
	  Stream Cipher Project. See <http://www.ecrypt.eu.org/stream/>

	  The Salsa20 stream cipher algorithm is designed by Daniel J.
	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>

config CRYPTO_SEED
	tristate "SEED cipher algorithm"
	select CRYPTO_ALGAPI
	help
	  SEED cipher algorithm (RFC4269).

	  SEED is a 128-bit symmetric key block cipher that has been
	  developed by KISA (Korea Information Security Agency) as a
	  national standard encryption algorithm of the Republic of Korea.
	  It is a 16 round block cipher with the key size of 128 bit.

	  See also:
	  <http://www.kisa.or.kr/kisa/seed/jsp/seed_eng.jsp>

config CRYPTO_SERPENT
	tristate "Serpent cipher algorithm"
	select CRYPTO_ALGAPI
	help
	  Serpent cipher algorithm, by Anderson, Biham & Knudsen.

	  Keys are allowed to be from 0 to 256 bits in length, in steps
	  of 8 bits.  Also includes the 'Tnepres' algorithm, a reversed
	  variant of Serpent for compatibility with old kerneli.org code.

	  See also:
	  <http://www.cl.cam.ac.uk/~rja14/serpent.html>

config CRYPTO_SERPENT_SSE2_X86_64
	tristate "Serpent cipher algorithm (x86_64/SSE2)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_SERPENT
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Serpent cipher algorithm, by Anderson, Biham & Knudsen.

	  Keys are allowed to be from 0 to 256 bits in length, in steps
	  of 8 bits.

	  This module provides Serpent cipher algorithm that processes eight
	  blocks parallel using SSE2 instruction set.

	  See also:
	  <http://www.cl.cam.ac.uk/~rja14/serpent.html>

config CRYPTO_SERPENT_SSE2_586
	tristate "Serpent cipher algorithm (i586/SSE2)"
	depends on X86 && !64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_SERPENT
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Serpent cipher algorithm, by Anderson, Biham & Knudsen.

	  Keys are allowed to be from 0 to 256 bits in length, in steps
	  of 8 bits.

	  This module provides Serpent cipher algorithm that processes four
	  blocks parallel using SSE2 instruction set.

	  See also:
	  <http://www.cl.cam.ac.uk/~rja14/serpent.html>

config CRYPTO_SERPENT_AVX_X86_64
	tristate "Serpent cipher algorithm (x86_64/AVX)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_SERPENT
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Serpent cipher algorithm, by Anderson, Biham & Knudsen.

	  Keys are allowed to be from 0 to 256 bits in length, in steps
	  of 8 bits.

	  This module provides the Serpent cipher algorithm that processes
	  eight blocks parallel using the AVX instruction set.

	  See also:
	  <http://www.cl.cam.ac.uk/~rja14/serpent.html>

config CRYPTO_SERPENT_AVX2_X86_64
	tristate "Serpent cipher algorithm (x86_64/AVX2)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_SERPENT
	select CRYPTO_SERPENT_AVX_X86_64
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Serpent cipher algorithm, by Anderson, Biham & Knudsen.

	  Keys are allowed to be from 0 to 256 bits in length, in steps
	  of 8 bits.

	  This module provides Serpent cipher algorithm that processes 16
	  blocks parallel using AVX2 instruction set.

	  See also:
	  <http://www.cl.cam.ac.uk/~rja14/serpent.html>

config CRYPTO_TEA
	tristate "TEA, XTEA and XETA cipher algorithms"
	select CRYPTO_ALGAPI
	help
	  TEA cipher algorithm.

	  Tiny Encryption Algorithm is a simple cipher that uses
	  many rounds for security.  It is very fast and uses
	  little memory.

	  Xtendend Tiny Encryption Algorithm is a modification to
	  the TEA algorithm to address a potential key weakness
	  in the TEA algorithm.

	  Xtendend Encryption Tiny Algorithm is a mis-implementation
	  of the XTEA algorithm for compatibility purposes.

config CRYPTO_TWOFISH
	tristate "Twofish cipher algorithm"
	select CRYPTO_ALGAPI
	select CRYPTO_TWOFISH_COMMON
	help
	  Twofish cipher algorithm.

	  Twofish was submitted as an AES (Advanced Encryption Standard)
	  candidate cipher by researchers at CounterPane Systems.  It is a
	  16 round block cipher supporting key sizes of 128, 192, and 256
	  bits.

	  See also:
	  <http://www.schneier.com/twofish.html>

config CRYPTO_TWOFISH_COMMON
	tristate
	help
	  Common parts of the Twofish cipher algorithm shared by the
	  generic c and the assembler implementations.

config CRYPTO_TWOFISH_586
	tristate "Twofish cipher algorithms (i586)"
	depends on (X86 || UML_X86) && !64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_TWOFISH_COMMON
	help
	  Twofish cipher algorithm.

	  Twofish was submitted as an AES (Advanced Encryption Standard)
	  candidate cipher by researchers at CounterPane Systems.  It is a
	  16 round block cipher supporting key sizes of 128, 192, and 256
	  bits.

	  See also:
	  <http://www.schneier.com/twofish.html>

config CRYPTO_TWOFISH_X86_64
	tristate "Twofish cipher algorithm (x86_64)"
	depends on (X86 || UML_X86) && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_TWOFISH_COMMON
	help
	  Twofish cipher algorithm (x86_64).

	  Twofish was submitted as an AES (Advanced Encryption Standard)
	  candidate cipher by researchers at CounterPane Systems.  It is a
	  16 round block cipher supporting key sizes of 128, 192, and 256
	  bits.

	  See also:
	  <http://www.schneier.com/twofish.html>

config CRYPTO_TWOFISH_X86_64_3WAY
	tristate "Twofish cipher algorithm (x86_64, 3-way parallel)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_TWOFISH_COMMON
	select CRYPTO_TWOFISH_X86_64
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Twofish cipher algorithm (x86_64, 3-way parallel).

	  Twofish was submitted as an AES (Advanced Encryption Standard)
	  candidate cipher by researchers at CounterPane Systems.  It is a
	  16 round block cipher supporting key sizes of 128, 192, and 256
	  bits.

	  This module provides Twofish cipher algorithm that processes three
	  blocks parallel, utilizing resources of out-of-order CPUs better.

	  See also:
	  <http://www.schneier.com/twofish.html>

config CRYPTO_TWOFISH_AVX_X86_64
	tristate "Twofish cipher algorithm (x86_64/AVX)"
	depends on X86 && 64BIT
	select CRYPTO_ALGAPI
	select CRYPTO_CRYPTD
	select CRYPTO_ABLK_HELPER
	select CRYPTO_GLUE_HELPER_X86
	select CRYPTO_TWOFISH_COMMON
	select CRYPTO_TWOFISH_X86_64
	select CRYPTO_TWOFISH_X86_64_3WAY
	select CRYPTO_LRW
	select CRYPTO_XTS
	help
	  Twofish cipher algorithm (x86_64/AVX).

	  Twofish was submitted as an AES (Advanced Encryption Standard)
	  candidate cipher by researchers at CounterPane Systems.  It is a
	  16 round block cipher supporting key sizes of 128, 192, and 256
	  bits.

	  This module provides the Twofish cipher algorithm that processes
	  eight blocks parallel using the AVX Instruction Set.

	  See also:
	  <http://www.schneier.com/twofish.html>

comment "Compression"

config CRYPTO_DEFLATE
	tristate "Deflate compression algorithm"
	select CRYPTO_ALGAPI
	select ZLIB_INFLATE
	select ZLIB_DEFLATE
	help
	  This is the Deflate algorithm (RFC1951), specified for use in
	  IPSec with the IPCOMP protocol (RFC3173, RFC2394).

	  You will most probably want this if using IPSec.

config CRYPTO_ZLIB
	tristate "Zlib compression algorithm"
	select CRYPTO_PCOMP
	select ZLIB_INFLATE
	select ZLIB_DEFLATE
	select NLATTR
	help
	  This is the zlib algorithm.

config CRYPTO_LZO
	tristate "LZO compression algorithm"
	select CRYPTO_ALGAPI
	select LZO_COMPRESS
	select LZO_DECOMPRESS
	help
	  This is the LZO algorithm.

config CRYPTO_842
	tristate "842 compression algorithm"
	depends on CRYPTO_DEV_NX_COMPRESS
	# 842 uses lzo if the hardware becomes unavailable
	select LZO_COMPRESS
	select LZO_DECOMPRESS
	help
	  This is the 842 algorithm.

config CRYPTO_LZ4
	tristate "LZ4 compression algorithm"
	select CRYPTO_ALGAPI
	select LZ4_COMPRESS
	select LZ4_DECOMPRESS
	help
	  This is the LZ4 algorithm.

config CRYPTO_LZ4HC
	tristate "LZ4HC compression algorithm"
	select CRYPTO_ALGAPI
	select LZ4HC_COMPRESS
	select LZ4_DECOMPRESS
	help
	  This is the LZ4 high compression mode algorithm.

comment "Random Number Generation"

config CRYPTO_ANSI_CPRNG
	tristate "Pseudo Random Number Generation for Cryptographic modules"
	default m
	select CRYPTO_AES
	select CRYPTO_RNG
	help
	  This option enables the generic pseudo random number generator
	  for cryptographic modules.  Uses the Algorithm specified in
	  ANSI X9.31 A.2.4. Note that this option must be enabled if
	  CRYPTO_FIPS is selected

menuconfig CRYPTO_DRBG_MENU
	tristate "NIST SP800-90A DRBG"
	help
	  NIST SP800-90A compliant DRBG. In the following submenu, one or
	  more of the DRBG types must be selected.

if CRYPTO_DRBG_MENU

config CRYPTO_DRBG_HMAC
	bool "Enable HMAC DRBG"
	default y
	select CRYPTO_HMAC
	help
	  Enable the HMAC DRBG variant as defined in NIST SP800-90A.

config CRYPTO_DRBG_HASH
	bool "Enable Hash DRBG"
	select CRYPTO_HASH
	help
	  Enable the Hash DRBG variant as defined in NIST SP800-90A.

config CRYPTO_DRBG_CTR
	bool "Enable CTR DRBG"
	select CRYPTO_AES
	help
	  Enable the CTR DRBG variant as defined in NIST SP800-90A.

config CRYPTO_DRBG
	tristate
	default CRYPTO_DRBG_MENU if (CRYPTO_DRBG_HMAC || CRYPTO_DRBG_HASH || CRYPTO_DRBG_CTR)
	select CRYPTO_RNG

endif	# if CRYPTO_DRBG_MENU

config CRYPTO_USER_API
	tristate

config CRYPTO_USER_API_HASH
	tristate "User-space interface for hash algorithms"
	depends on NET
	select CRYPTO_HASH
	select CRYPTO_USER_API
	help
	  This option enables the user-spaces interface for hash
	  algorithms.

config CRYPTO_USER_API_SKCIPHER
	tristate "User-space interface for symmetric key cipher algorithms"
	depends on NET
	select CRYPTO_BLKCIPHER
	select CRYPTO_USER_API
	help
	  This option enables the user-spaces interface for symmetric
	  key cipher algorithms.

config CRYPTO_USER_API_RNG
	tristate "User-space interface for random number generator algorithms"
	depends on NET
	select CRYPTO_RNG
	select CRYPTO_USER_API
	help
	  This option enables the user-spaces interface for random
	  number generator algorithms.

config CRYPTO_USER_API_AEAD
	tristate "User-space interface for AEAD cipher algorithms"
	depends on NET
	select CRYPTO_AEAD
	select CRYPTO_USER_API
	help
	  This option enables the user-spaces interface for AEAD
	  cipher algorithms.

config CRYPTO_HASH_INFO
	bool

source "drivers/crypto/Kconfig"
source crypto/asymmetric_keys/Kconfig

endif	# if CRYPTO
a> 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Broadcom GENET (Gigabit Ethernet) controller driver
 *
 * Copyright (c) 2014-2017 Broadcom
 */

#define pr_fmt(fmt)				"bcmgenet: " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/interrupt.h>
#include <linux/string.h>
#include <linux/if_ether.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/delay.h>
#include <linux/platform_device.h>
#include <linux/dma-mapping.h>
#include <linux/pm.h>
#include <linux/clk.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/of_net.h>
#include <linux/of_platform.h>
#include <net/arp.h>

#include <linux/mii.h>
#include <linux/ethtool.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/phy.h>
#include <linux/platform_data/bcmgenet.h>

#include <asm/unaligned.h>

#include "bcmgenet.h"

/* Maximum number of hardware queues, downsized if needed */
#define GENET_MAX_MQ_CNT	4

/* Default highest priority queue for multi queue support */
#define GENET_Q0_PRIORITY	0

#define GENET_Q16_RX_BD_CNT	\
	(TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
#define GENET_Q16_TX_BD_CNT	\
	(TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)

#define RX_BUF_LENGTH		2048
#define SKB_ALIGNMENT		32

/* Tx/Rx DMA register offset, skip 256 descriptors */
#define WORDS_PER_BD(p)		(p->hw_params->words_per_bd)
#define DMA_DESC_SIZE		(WORDS_PER_BD(priv) * sizeof(u32))

#define GENET_TDMA_REG_OFF	(priv->hw_params->tdma_offset + \
				TOTAL_DESC * DMA_DESC_SIZE)

#define GENET_RDMA_REG_OFF	(priv->hw_params->rdma_offset + \
				TOTAL_DESC * DMA_DESC_SIZE)

/* Forward declarations */
static void bcmgenet_set_rx_mode(struct net_device *dev);

static inline void bcmgenet_writel(u32 value, void __iomem *offset)
{
	/* MIPS chips strapped for BE will automagically configure the
	 * peripheral registers for CPU-native byte order.
	 */
	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
		__raw_writel(value, offset);
	else
		writel_relaxed(value, offset);
}

static inline u32 bcmgenet_readl(void __iomem *offset)
{
	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
		return __raw_readl(offset);
	else
		return readl_relaxed(offset);
}

static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
					     void __iomem *d, u32 value)
{
	bcmgenet_writel(value, d + DMA_DESC_LENGTH_STATUS);
}

static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv,
					    void __iomem *d)
{
	return bcmgenet_readl(d + DMA_DESC_LENGTH_STATUS);
}

static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
				    void __iomem *d,
				    dma_addr_t addr)
{
	bcmgenet_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);

	/* Register writes to GISB bus can take couple hundred nanoseconds
	 * and are done for each packet, save these expensive writes unless
	 * the platform is explicitly configured for 64-bits/LPAE.
	 */
#ifdef CONFIG_PHYS_ADDR_T_64BIT
	if (priv->hw_params->flags & GENET_HAS_40BITS)
		bcmgenet_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
#endif
}

/* Combined address + length/status setter */
static inline void dmadesc_set(struct bcmgenet_priv *priv,
			       void __iomem *d, dma_addr_t addr, u32 val)
{
	dmadesc_set_addr(priv, d, addr);
	dmadesc_set_length_status(priv, d, val);
}

static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
					  void __iomem *d)
{
	dma_addr_t addr;

	addr = bcmgenet_readl(d + DMA_DESC_ADDRESS_LO);

	/* Register writes to GISB bus can take couple hundred nanoseconds
	 * and are done for each packet, save these expensive writes unless
	 * the platform is explicitly configured for 64-bits/LPAE.
	 */
#ifdef CONFIG_PHYS_ADDR_T_64BIT
	if (priv->hw_params->flags & GENET_HAS_40BITS)
		addr |= (u64)bcmgenet_readl(d + DMA_DESC_ADDRESS_HI) << 32;
#endif
	return addr;
}

#define GENET_VER_FMT	"%1d.%1d EPHY: 0x%04x"

#define GENET_MSG_DEFAULT	(NETIF_MSG_DRV | NETIF_MSG_PROBE | \
				NETIF_MSG_LINK)

static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
{
	if (GENET_IS_V1(priv))
		return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
	else
		return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
}

static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
{
	if (GENET_IS_V1(priv))
		bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
	else
		bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
}

/* These macros are defined to deal with register map change
 * between GENET1.1 and GENET2. Only those currently being used
 * by driver are defined.
 */
static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
{
	if (GENET_IS_V1(priv))
		return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
	else
		return bcmgenet_readl(priv->base +
				      priv->hw_params->tbuf_offset + TBUF_CTRL);
}

static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
{
	if (GENET_IS_V1(priv))
		bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
	else
		bcmgenet_writel(val, priv->base +
				priv->hw_params->tbuf_offset + TBUF_CTRL);
}

static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
{
	if (GENET_IS_V1(priv))
		return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
	else
		return bcmgenet_readl(priv->base +
				      priv->hw_params->tbuf_offset + TBUF_BP_MC);
}

static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
{
	if (GENET_IS_V1(priv))
		bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
	else
		bcmgenet_writel(val, priv->base +
				priv->hw_params->tbuf_offset + TBUF_BP_MC);
}

/* RX/TX DMA register accessors */
enum dma_reg {
	DMA_RING_CFG = 0,
	DMA_CTRL,
	DMA_STATUS,
	DMA_SCB_BURST_SIZE,
	DMA_ARB_CTRL,
	DMA_PRIORITY_0,
	DMA_PRIORITY_1,
	DMA_PRIORITY_2,
	DMA_INDEX2RING_0,
	DMA_INDEX2RING_1,
	DMA_INDEX2RING_2,
	DMA_INDEX2RING_3,
	DMA_INDEX2RING_4,
	DMA_INDEX2RING_5,
	DMA_INDEX2RING_6,
	DMA_INDEX2RING_7,
	DMA_RING0_TIMEOUT,
	DMA_RING1_TIMEOUT,
	DMA_RING2_TIMEOUT,
	DMA_RING3_TIMEOUT,
	DMA_RING4_TIMEOUT,
	DMA_RING5_TIMEOUT,
	DMA_RING6_TIMEOUT,
	DMA_RING7_TIMEOUT,
	DMA_RING8_TIMEOUT,
	DMA_RING9_TIMEOUT,
	DMA_RING10_TIMEOUT,
	DMA_RING11_TIMEOUT,
	DMA_RING12_TIMEOUT,
	DMA_RING13_TIMEOUT,
	DMA_RING14_TIMEOUT,
	DMA_RING15_TIMEOUT,
	DMA_RING16_TIMEOUT,
};

static const u8 bcmgenet_dma_regs_v3plus[] = {
	[DMA_RING_CFG]		= 0x00,
	[DMA_CTRL]		= 0x04,
	[DMA_STATUS]		= 0x08,
	[DMA_SCB_BURST_SIZE]	= 0x0C,
	[DMA_ARB_CTRL]		= 0x2C,
	[DMA_PRIORITY_0]	= 0x30,
	[DMA_PRIORITY_1]	= 0x34,
	[DMA_PRIORITY_2]	= 0x38,
	[DMA_RING0_TIMEOUT]	= 0x2C,
	[DMA_RING1_TIMEOUT]	= 0x30,
	[DMA_RING2_TIMEOUT]	= 0x34,
	[DMA_RING3_TIMEOUT]	= 0x38,
	[DMA_RING4_TIMEOUT]	= 0x3c,
	[DMA_RING5_TIMEOUT]	= 0x40,
	[DMA_RING6_TIMEOUT]	= 0x44,
	[DMA_RING7_TIMEOUT]	= 0x48,
	[DMA_RING8_TIMEOUT]	= 0x4c,
	[DMA_RING9_TIMEOUT]	= 0x50,
	[DMA_RING10_TIMEOUT]	= 0x54,
	[DMA_RING11_TIMEOUT]	= 0x58,
	[DMA_RING12_TIMEOUT]	= 0x5c,
	[DMA_RING13_TIMEOUT]	= 0x60,
	[DMA_RING14_TIMEOUT]	= 0x64,
	[DMA_RING15_TIMEOUT]	= 0x68,
	[DMA_RING16_TIMEOUT]	= 0x6C,
	[DMA_INDEX2RING_0]	= 0x70,
	[DMA_INDEX2RING_1]	= 0x74,
	[DMA_INDEX2RING_2]	= 0x78,
	[DMA_INDEX2RING_3]	= 0x7C,
	[DMA_INDEX2RING_4]	= 0x80,
	[DMA_INDEX2RING_5]	= 0x84,
	[DMA_INDEX2RING_6]	= 0x88,
	[DMA_INDEX2RING_7]	= 0x8C,
};

static const u8 bcmgenet_dma_regs_v2[] = {
	[DMA_RING_CFG]		= 0x00,
	[DMA_CTRL]		= 0x04,
	[DMA_STATUS]		= 0x08,
	[DMA_SCB_BURST_SIZE]	= 0x0C,
	[DMA_ARB_CTRL]		= 0x30,
	[DMA_PRIORITY_0]	= 0x34,
	[DMA_PRIORITY_1]	= 0x38,
	[DMA_PRIORITY_2]	= 0x3C,
	[DMA_RING0_TIMEOUT]	= 0x2C,
	[DMA_RING1_TIMEOUT]	= 0x30,
	[DMA_RING2_TIMEOUT]	= 0x34,
	[DMA_RING3_TIMEOUT]	= 0x38,
	[DMA_RING4_TIMEOUT]	= 0x3c,
	[DMA_RING5_TIMEOUT]	= 0x40,
	[DMA_RING6_TIMEOUT]	= 0x44,
	[DMA_RING7_TIMEOUT]	= 0x48,
	[DMA_RING8_TIMEOUT]	= 0x4c,
	[DMA_RING9_TIMEOUT]	= 0x50,
	[DMA_RING10_TIMEOUT]	= 0x54,
	[DMA_RING11_TIMEOUT]	= 0x58,
	[DMA_RING12_TIMEOUT]	= 0x5c,
	[DMA_RING13_TIMEOUT]	= 0x60,
	[DMA_RING14_TIMEOUT]	= 0x64,
	[DMA_RING15_TIMEOUT]	= 0x68,
	[DMA_RING16_TIMEOUT]	= 0x6C,
};

static const u8 bcmgenet_dma_regs_v1[] = {
	[DMA_CTRL]		= 0x00,
	[DMA_STATUS]		= 0x04,
	[DMA_SCB_BURST_SIZE]	= 0x0C,
	[DMA_ARB_CTRL]		= 0x30,
	[DMA_PRIORITY_0]	= 0x34,
	[DMA_PRIORITY_1]	= 0x38,
	[DMA_PRIORITY_2]	= 0x3C,
	[DMA_RING0_TIMEOUT]	= 0x2C,
	[DMA_RING1_TIMEOUT]	= 0x30,
	[DMA_RING2_TIMEOUT]	= 0x34,
	[DMA_RING3_TIMEOUT]	= 0x38,
	[DMA_RING4_TIMEOUT]	= 0x3c,
	[DMA_RING5_TIMEOUT]	= 0x40,
	[DMA_RING6_TIMEOUT]	= 0x44,
	[DMA_RING7_TIMEOUT]	= 0x48,
	[DMA_RING8_TIMEOUT]	= 0x4c,
	[DMA_RING9_TIMEOUT]	= 0x50,
	[DMA_RING10_TIMEOUT]	= 0x54,
	[DMA_RING11_TIMEOUT]	= 0x58,
	[DMA_RING12_TIMEOUT]	= 0x5c,
	[DMA_RING13_TIMEOUT]	= 0x60,
	[DMA_RING14_TIMEOUT]	= 0x64,
	[DMA_RING15_TIMEOUT]	= 0x68,
	[DMA_RING16_TIMEOUT]	= 0x6C,
};

/* Set at runtime once bcmgenet version is known */
static const u8 *bcmgenet_dma_regs;

static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
{
	return netdev_priv(dev_get_drvdata(dev));
}

static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
				      enum dma_reg r)
{
	return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
			      DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
}

static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
					u32 val, enum dma_reg r)
{
	bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
}

static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
				      enum dma_reg r)
{
	return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
			      DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
}

static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
					u32 val, enum dma_reg r)
{
	bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
}

/* RDMA/TDMA ring registers and accessors
 * we merge the common fields and just prefix with T/D the registers
 * having different meaning depending on the direction
 */
enum dma_ring_reg {
	TDMA_READ_PTR = 0,
	RDMA_WRITE_PTR = TDMA_READ_PTR,
	TDMA_READ_PTR_HI,
	RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
	TDMA_CONS_INDEX,
	RDMA_PROD_INDEX = TDMA_CONS_INDEX,
	TDMA_PROD_INDEX,
	RDMA_CONS_INDEX = TDMA_PROD_INDEX,
	DMA_RING_BUF_SIZE,
	DMA_START_ADDR,
	DMA_START_ADDR_HI,
	DMA_END_ADDR,
	DMA_END_ADDR_HI,
	DMA_MBUF_DONE_THRESH,
	TDMA_FLOW_PERIOD,
	RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
	TDMA_WRITE_PTR,
	RDMA_READ_PTR = TDMA_WRITE_PTR,
	TDMA_WRITE_PTR_HI,
	RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
};

/* GENET v4 supports 40-bits pointer addressing
 * for obvious reasons the LO and HI word parts
 * are contiguous, but this offsets the other
 * registers.
 */
static const u8 genet_dma_ring_regs_v4[] = {
	[TDMA_READ_PTR]			= 0x00,
	[TDMA_READ_PTR_HI]		= 0x04,
	[TDMA_CONS_INDEX]		= 0x08,
	[TDMA_PROD_INDEX]		= 0x0C,
	[DMA_RING_BUF_SIZE]		= 0x10,
	[DMA_START_ADDR]		= 0x14,
	[DMA_START_ADDR_HI]		= 0x18,
	[DMA_END_ADDR]			= 0x1C,
	[DMA_END_ADDR_HI]		= 0x20,
	[DMA_MBUF_DONE_THRESH]		= 0x24,
	[TDMA_FLOW_PERIOD]		= 0x28,
	[TDMA_WRITE_PTR]		= 0x2C,
	[TDMA_WRITE_PTR_HI]		= 0x30,
};

static const u8 genet_dma_ring_regs_v123[] = {
	[TDMA_READ_PTR]			= 0x00,
	[TDMA_CONS_INDEX]		= 0x04,
	[TDMA_PROD_INDEX]		= 0x08,
	[DMA_RING_BUF_SIZE]		= 0x0C,
	[DMA_START_ADDR]		= 0x10,
	[DMA_END_ADDR]			= 0x14,
	[DMA_MBUF_DONE_THRESH]		= 0x18,
	[TDMA_FLOW_PERIOD]		= 0x1C,
	[TDMA_WRITE_PTR]		= 0x20,
};

/* Set at runtime once GENET version is known */
static const u8 *genet_dma_ring_regs;

static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
					   unsigned int ring,
					   enum dma_ring_reg r)
{
	return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
			      (DMA_RING_SIZE * ring) +
			      genet_dma_ring_regs[r]);
}

static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
					     unsigned int ring, u32 val,
					     enum dma_ring_reg r)
{
	bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
			(DMA_RING_SIZE * ring) +
			genet_dma_ring_regs[r]);
}

static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
					   unsigned int ring,
					   enum dma_ring_reg r)
{
	return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
			      (DMA_RING_SIZE * ring) +
			      genet_dma_ring_regs[r]);
}

static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
					     unsigned int ring, u32 val,
					     enum dma_ring_reg r)
{
	bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
			(DMA_RING_SIZE * ring) +
			genet_dma_ring_regs[r]);
}

static int bcmgenet_begin(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	/* Turn on the clock */
	return clk_prepare_enable(priv->clk);
}

static void bcmgenet_complete(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	/* Turn off the clock */
	clk_disable_unprepare(priv->clk);
}

static int bcmgenet_get_link_ksettings(struct net_device *dev,
				       struct ethtool_link_ksettings *cmd)
{
	if (!netif_running(dev))
		return -EINVAL;

	if (!dev->phydev)
		return -ENODEV;

	phy_ethtool_ksettings_get(dev->phydev, cmd);

	return 0;
}

static int bcmgenet_set_link_ksettings(struct net_device *dev,
				       const struct ethtool_link_ksettings *cmd)
{
	if (!netif_running(dev))
		return -EINVAL;

	if (!dev->phydev)
		return -ENODEV;

	return phy_ethtool_ksettings_set(dev->phydev, cmd);
}

static int bcmgenet_set_rx_csum(struct net_device *dev,
				netdev_features_t wanted)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	u32 rbuf_chk_ctrl;
	bool rx_csum_en;

	rx_csum_en = !!(wanted & NETIF_F_RXCSUM);

	rbuf_chk_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);

	/* enable rx checksumming */
	if (rx_csum_en)
		rbuf_chk_ctrl |= RBUF_RXCHK_EN;
	else
		rbuf_chk_ctrl &= ~RBUF_RXCHK_EN;
	priv->desc_rxchk_en = rx_csum_en;

	/* If UniMAC forwards CRC, we need to skip over it to get
	 * a valid CHK bit to be set in the per-packet status word
	*/
	if (rx_csum_en && priv->crc_fwd_en)
		rbuf_chk_ctrl |= RBUF_SKIP_FCS;
	else
		rbuf_chk_ctrl &= ~RBUF_SKIP_FCS;

	bcmgenet_rbuf_writel(priv, rbuf_chk_ctrl, RBUF_CHK_CTRL);

	return 0;
}

static int bcmgenet_set_tx_csum(struct net_device *dev,
				netdev_features_t wanted)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	bool desc_64b_en;
	u32 tbuf_ctrl, rbuf_ctrl;

	tbuf_ctrl = bcmgenet_tbuf_ctrl_get(priv);
	rbuf_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CTRL);

	desc_64b_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));

	/* enable 64 bytes descriptor in both directions (RBUF and TBUF) */
	if (desc_64b_en) {
		tbuf_ctrl |= RBUF_64B_EN;
		rbuf_ctrl |= RBUF_64B_EN;
	} else {
		tbuf_ctrl &= ~RBUF_64B_EN;
		rbuf_ctrl &= ~RBUF_64B_EN;
	}
	priv->desc_64b_en = desc_64b_en;

	bcmgenet_tbuf_ctrl_set(priv, tbuf_ctrl);
	bcmgenet_rbuf_writel(priv, rbuf_ctrl, RBUF_CTRL);

	return 0;
}

static int bcmgenet_set_features(struct net_device *dev,
				 netdev_features_t features)
{
	netdev_features_t changed = features ^ dev->features;
	netdev_features_t wanted = dev->wanted_features;
	int ret = 0;

	if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
		ret = bcmgenet_set_tx_csum(dev, wanted);
	if (changed & (NETIF_F_RXCSUM))
		ret = bcmgenet_set_rx_csum(dev, wanted);

	return ret;
}

static u32 bcmgenet_get_msglevel(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	return priv->msg_enable;
}

static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	priv->msg_enable = level;
}

static int bcmgenet_get_coalesce(struct net_device *dev,
				 struct ethtool_coalesce *ec)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	struct bcmgenet_rx_ring *ring;
	unsigned int i;

	ec->tx_max_coalesced_frames =
		bcmgenet_tdma_ring_readl(priv, DESC_INDEX,
					 DMA_MBUF_DONE_THRESH);
	ec->rx_max_coalesced_frames =
		bcmgenet_rdma_ring_readl(priv, DESC_INDEX,
					 DMA_MBUF_DONE_THRESH);
	ec->rx_coalesce_usecs =
		bcmgenet_rdma_readl(priv, DMA_RING16_TIMEOUT) * 8192 / 1000;

	for (i = 0; i < priv->hw_params->rx_queues; i++) {
		ring = &priv->rx_rings[i];
		ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;
	}
	ring = &priv->rx_rings[DESC_INDEX];
	ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;

	return 0;
}

static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring,
				     u32 usecs, u32 pkts)
{
	struct bcmgenet_priv *priv = ring->priv;
	unsigned int i = ring->index;
	u32 reg;

	bcmgenet_rdma_ring_writel(priv, i, pkts, DMA_MBUF_DONE_THRESH);

	reg = bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT + i);
	reg &= ~DMA_TIMEOUT_MASK;
	reg |= DIV_ROUND_UP(usecs * 1000, 8192);
	bcmgenet_rdma_writel(priv, reg, DMA_RING0_TIMEOUT + i);
}

static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring,
					  struct ethtool_coalesce *ec)
{
	struct dim_cq_moder moder;
	u32 usecs, pkts;

	ring->rx_coalesce_usecs = ec->rx_coalesce_usecs;
	ring->rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
	usecs = ring->rx_coalesce_usecs;
	pkts = ring->rx_max_coalesced_frames;

	if (ec->use_adaptive_rx_coalesce && !ring->dim.use_dim) {
		moder = net_dim_get_def_rx_moderation(ring->dim.dim.mode);
		usecs = moder.usec;
		pkts = moder.pkts;
	}

	ring->dim.use_dim = ec->use_adaptive_rx_coalesce;
	bcmgenet_set_rx_coalesce(ring, usecs, pkts);
}

static int bcmgenet_set_coalesce(struct net_device *dev,
				 struct ethtool_coalesce *ec)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	unsigned int i;

	/* Base system clock is 125Mhz, DMA timeout is this reference clock
	 * divided by 1024, which yields roughly 8.192us, our maximum value
	 * has to fit in the DMA_TIMEOUT_MASK (16 bits)
	 */
	if (ec->tx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
	    ec->tx_max_coalesced_frames == 0 ||
	    ec->rx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
	    ec->rx_coalesce_usecs > (DMA_TIMEOUT_MASK * 8) + 1)
		return -EINVAL;

	if (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0)
		return -EINVAL;

	/* GENET TDMA hardware does not support a configurable timeout, but will
	 * always generate an interrupt either after MBDONE packets have been
	 * transmitted, or when the ring is empty.
	 */
	if (ec->tx_coalesce_usecs || ec->tx_coalesce_usecs_high ||
	    ec->tx_coalesce_usecs_irq || ec->tx_coalesce_usecs_low ||
	    ec->use_adaptive_tx_coalesce)
		return -EOPNOTSUPP;

	/* Program all TX queues with the same values, as there is no
	 * ethtool knob to do coalescing on a per-queue basis
	 */
	for (i = 0; i < priv->hw_params->tx_queues; i++)
		bcmgenet_tdma_ring_writel(priv, i,
					  ec->tx_max_coalesced_frames,
					  DMA_MBUF_DONE_THRESH);
	bcmgenet_tdma_ring_writel(priv, DESC_INDEX,
				  ec->tx_max_coalesced_frames,
				  DMA_MBUF_DONE_THRESH);

	for (i = 0; i < priv->hw_params->rx_queues; i++)
		bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[i], ec);
	bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[DESC_INDEX], ec);

	return 0;
}

/* standard ethtool support functions. */
enum bcmgenet_stat_type {
	BCMGENET_STAT_NETDEV = -1,
	BCMGENET_STAT_MIB_RX,
	BCMGENET_STAT_MIB_TX,
	BCMGENET_STAT_RUNT,
	BCMGENET_STAT_MISC,
	BCMGENET_STAT_SOFT,
};

struct bcmgenet_stats {
	char stat_string[ETH_GSTRING_LEN];
	int stat_sizeof;
	int stat_offset;
	enum bcmgenet_stat_type type;
	/* reg offset from UMAC base for misc counters */
	u16 reg_offset;
};

#define STAT_NETDEV(m) { \
	.stat_string = __stringify(m), \
	.stat_sizeof = sizeof(((struct net_device_stats *)0)->m), \
	.stat_offset = offsetof(struct net_device_stats, m), \
	.type = BCMGENET_STAT_NETDEV, \
}

#define STAT_GENET_MIB(str, m, _type) { \
	.stat_string = str, \
	.stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
	.stat_offset = offsetof(struct bcmgenet_priv, m), \
	.type = _type, \
}

#define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
#define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
#define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
#define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)

#define STAT_GENET_MISC(str, m, offset) { \
	.stat_string = str, \
	.stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
	.stat_offset = offsetof(struct bcmgenet_priv, m), \
	.type = BCMGENET_STAT_MISC, \
	.reg_offset = offset, \
}

#define STAT_GENET_Q(num) \
	STAT_GENET_SOFT_MIB("txq" __stringify(num) "_packets", \
			tx_rings[num].packets), \
	STAT_GENET_SOFT_MIB("txq" __stringify(num) "_bytes", \
			tx_rings[num].bytes), \
	STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_bytes", \
			rx_rings[num].bytes),	 \
	STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_packets", \
			rx_rings[num].packets), \
	STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_errors", \
			rx_rings[num].errors), \
	STAT_GENET_SOFT_MIB("rxq" __stringify(num) "_dropped", \
			rx_rings[num].dropped)

/* There is a 0xC gap between the end of RX and beginning of TX stats and then
 * between the end of TX stats and the beginning of the RX RUNT
 */
#define BCMGENET_STAT_OFFSET	0xc

/* Hardware counters must be kept in sync because the order/offset
 * is important here (order in structure declaration = order in hardware)
 */
static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
	/* general stats */
	STAT_NETDEV(rx_packets),
	STAT_NETDEV(tx_packets),
	STAT_NETDEV(rx_bytes),
	STAT_NETDEV(tx_bytes),
	STAT_NETDEV(rx_errors),
	STAT_NETDEV(tx_errors),
	STAT_NETDEV(rx_dropped),
	STAT_NETDEV(tx_dropped),
	STAT_NETDEV(multicast),
	/* UniMAC RSV counters */
	STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
	STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
	STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
	STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
	STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
	STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
	STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
	STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
	STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
	STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
	STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
	STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
	STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
	STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
	STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
	STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
	STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
	STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
	STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
	STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
	STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
	STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
	STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
	STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
	STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
	STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
	STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
	STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
	STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
	/* UniMAC TSV counters */
	STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
	STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
	STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
	STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
	STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
	STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
	STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
	STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
	STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
	STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
	STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
	STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
	STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
	STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
	STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
	STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
	STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
	STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
	STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
	STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
	STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
	STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
	STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
	STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
	STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
	STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
	STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
	STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
	STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
	/* UniMAC RUNT counters */
	STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
	STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
	STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
	STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
	/* Misc UniMAC counters */
	STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
			UMAC_RBUF_OVFL_CNT_V1),
	STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt,
			UMAC_RBUF_ERR_CNT_V1),
	STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
	STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
	STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed),
	STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
	/* Per TX queues */
	STAT_GENET_Q(0),
	STAT_GENET_Q(1),
	STAT_GENET_Q(2),
	STAT_GENET_Q(3),
	STAT_GENET_Q(16),
};

#define BCMGENET_STATS_LEN	ARRAY_SIZE(bcmgenet_gstrings_stats)

static void bcmgenet_get_drvinfo(struct net_device *dev,
				 struct ethtool_drvinfo *info)
{
	strlcpy(info->driver, "bcmgenet", sizeof(info->driver));
	strlcpy(info->version, "v2.0", sizeof(info->version));
}

static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
{
	switch (string_set) {
	case ETH_SS_STATS:
		return BCMGENET_STATS_LEN;
	default:
		return -EOPNOTSUPP;
	}
}

static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
				 u8 *data)
{
	int i;

	switch (stringset) {
	case ETH_SS_STATS:
		for (i = 0; i < BCMGENET_STATS_LEN; i++) {
			memcpy(data + i * ETH_GSTRING_LEN,
			       bcmgenet_gstrings_stats[i].stat_string,
			       ETH_GSTRING_LEN);
		}
		break;
	}
}

static u32 bcmgenet_update_stat_misc(struct bcmgenet_priv *priv, u16 offset)
{
	u16 new_offset;
	u32 val;

	switch (offset) {
	case UMAC_RBUF_OVFL_CNT_V1:
		if (GENET_IS_V2(priv))
			new_offset = RBUF_OVFL_CNT_V2;
		else
			new_offset = RBUF_OVFL_CNT_V3PLUS;

		val = bcmgenet_rbuf_readl(priv,	new_offset);
		/* clear if overflowed */
		if (val == ~0)
			bcmgenet_rbuf_writel(priv, 0, new_offset);
		break;
	case UMAC_RBUF_ERR_CNT_V1:
		if (GENET_IS_V2(priv))
			new_offset = RBUF_ERR_CNT_V2;
		else
			new_offset = RBUF_ERR_CNT_V3PLUS;

		val = bcmgenet_rbuf_readl(priv,	new_offset);
		/* clear if overflowed */
		if (val == ~0)
			bcmgenet_rbuf_writel(priv, 0, new_offset);
		break;
	default:
		val = bcmgenet_umac_readl(priv, offset);
		/* clear if overflowed */
		if (val == ~0)
			bcmgenet_umac_writel(priv, 0, offset);
		break;
	}

	return val;
}

static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
{
	int i, j = 0;

	for (i = 0; i < BCMGENET_STATS_LEN; i++) {
		const struct bcmgenet_stats *s;
		u8 offset = 0;
		u32 val = 0;
		char *p;

		s = &bcmgenet_gstrings_stats[i];
		switch (s->type) {
		case BCMGENET_STAT_NETDEV:
		case BCMGENET_STAT_SOFT:
			continue;
		case BCMGENET_STAT_RUNT:
			offset += BCMGENET_STAT_OFFSET;
			/* fall through */
		case BCMGENET_STAT_MIB_TX:
			offset += BCMGENET_STAT_OFFSET;
			/* fall through */
		case BCMGENET_STAT_MIB_RX:
			val = bcmgenet_umac_readl(priv,
						  UMAC_MIB_START + j + offset);
			offset = 0;	/* Reset Offset */
			break;
		case BCMGENET_STAT_MISC:
			if (GENET_IS_V1(priv)) {
				val = bcmgenet_umac_readl(priv, s->reg_offset);
				/* clear if overflowed */
				if (val == ~0)
					bcmgenet_umac_writel(priv, 0,
							     s->reg_offset);
			} else {
				val = bcmgenet_update_stat_misc(priv,
								s->reg_offset);
			}
			break;
		}

		j += s->stat_sizeof;
		p = (char *)priv + s->stat_offset;
		*(u32 *)p = val;
	}
}

static void bcmgenet_get_ethtool_stats(struct net_device *dev,
				       struct ethtool_stats *stats,
				       u64 *data)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	int i;

	if (netif_running(dev))
		bcmgenet_update_mib_counters(priv);

	dev->netdev_ops->ndo_get_stats(dev);

	for (i = 0; i < BCMGENET_STATS_LEN; i++) {
		const struct bcmgenet_stats *s;
		char *p;

		s = &bcmgenet_gstrings_stats[i];
		if (s->type == BCMGENET_STAT_NETDEV)
			p = (char *)&dev->stats;
		else
			p = (char *)priv;
		p += s->stat_offset;
		if (sizeof(unsigned long) != sizeof(u32) &&
		    s->stat_sizeof == sizeof(unsigned long))
			data[i] = *(unsigned long *)p;
		else
			data[i] = *(u32 *)p;
	}
}

void bcmgenet_eee_enable_set(struct net_device *dev, bool enable,
			     bool tx_lpi_enabled)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
	u32 reg;

	if (enable && !priv->clk_eee_enabled) {
		clk_prepare_enable(priv->clk_eee);
		priv->clk_eee_enabled = true;
	}

	reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
	if (enable)
		reg |= EEE_EN;
	else
		reg &= ~EEE_EN;
	bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);

	/* Enable EEE and switch to a 27Mhz clock automatically */
	reg = bcmgenet_readl(priv->base + off);
	if (tx_lpi_enabled)
		reg |= TBUF_EEE_EN | TBUF_PM_EN;
	else
		reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
	bcmgenet_writel(reg, priv->base + off);

	/* Do the same for thing for RBUF */
	reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
	if (enable)
		reg |= RBUF_EEE_EN | RBUF_PM_EN;
	else
		reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
	bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);

	if (!enable && priv->clk_eee_enabled) {
		clk_disable_unprepare(priv->clk_eee);
		priv->clk_eee_enabled = false;
	}

	priv->eee.eee_enabled = enable;
	priv->eee.eee_active = enable;
	priv->eee.tx_lpi_enabled = tx_lpi_enabled;
}

static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	struct ethtool_eee *p = &priv->eee;

	if (GENET_IS_V1(priv))
		return -EOPNOTSUPP;

	if (!dev->phydev)
		return -ENODEV;

	e->eee_enabled = p->eee_enabled;
	e->eee_active = p->eee_active;
	e->tx_lpi_enabled = p->tx_lpi_enabled;
	e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);

	return phy_ethtool_get_eee(dev->phydev, e);
}

static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	struct ethtool_eee *p = &priv->eee;

	if (GENET_IS_V1(priv))
		return -EOPNOTSUPP;

	if (!dev->phydev)
		return -ENODEV;

	p->eee_enabled = e->eee_enabled;

	if (!p->eee_enabled) {
		bcmgenet_eee_enable_set(dev, false, false);
	} else {
		p->eee_active = phy_init_eee(dev->phydev, false) >= 0;
		bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
		bcmgenet_eee_enable_set(dev, p->eee_active, e->tx_lpi_enabled);
	}

	return phy_ethtool_set_eee(dev->phydev, e);
}

/* standard ethtool support functions. */
static const struct ethtool_ops bcmgenet_ethtool_ops = {
	.begin			= bcmgenet_begin,
	.complete		= bcmgenet_complete,
	.get_strings		= bcmgenet_get_strings,
	.get_sset_count		= bcmgenet_get_sset_count,
	.get_ethtool_stats	= bcmgenet_get_ethtool_stats,
	.get_drvinfo		= bcmgenet_get_drvinfo,
	.get_link		= ethtool_op_get_link,
	.get_msglevel		= bcmgenet_get_msglevel,
	.set_msglevel		= bcmgenet_set_msglevel,
	.get_wol		= bcmgenet_get_wol,
	.set_wol		= bcmgenet_set_wol,
	.get_eee		= bcmgenet_get_eee,
	.set_eee		= bcmgenet_set_eee,
	.nway_reset		= phy_ethtool_nway_reset,
	.get_coalesce		= bcmgenet_get_coalesce,
	.set_coalesce		= bcmgenet_set_coalesce,
	.get_link_ksettings	= bcmgenet_get_link_ksettings,
	.set_link_ksettings	= bcmgenet_set_link_ksettings,
	.get_ts_info		= ethtool_op_get_ts_info,
};

/* Power down the unimac, based on mode. */
static int bcmgenet_power_down(struct bcmgenet_priv *priv,
				enum bcmgenet_power_mode mode)
{
	int ret = 0;
	u32 reg;

	switch (mode) {
	case GENET_POWER_CABLE_SENSE:
		phy_detach(priv->dev->phydev);
		break;

	case GENET_POWER_WOL_MAGIC:
		ret = bcmgenet_wol_power_down_cfg(priv, mode);
		break;

	case GENET_POWER_PASSIVE:
		/* Power down LED */
		if (priv->hw_params->flags & GENET_HAS_EXT) {
			reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
			if (GENET_IS_V5(priv))
				reg |= EXT_PWR_DOWN_PHY_EN |
				       EXT_PWR_DOWN_PHY_RD |
				       EXT_PWR_DOWN_PHY_SD |
				       EXT_PWR_DOWN_PHY_RX |
				       EXT_PWR_DOWN_PHY_TX |
				       EXT_IDDQ_GLBL_PWR;
			else
				reg |= EXT_PWR_DOWN_PHY;

			reg |= (EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
			bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);

			bcmgenet_phy_power_set(priv->dev, false);
		}
		break;
	default:
		break;
	}

	return ret;
}

static void bcmgenet_power_up(struct bcmgenet_priv *priv,
			      enum bcmgenet_power_mode mode)
{
	u32 reg;

	if (!(priv->hw_params->flags & GENET_HAS_EXT))
		return;

	reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);

	switch (mode) {
	case GENET_POWER_PASSIVE:
		reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS |
			 EXT_ENERGY_DET_MASK);
		if (GENET_IS_V5(priv)) {
			reg &= ~(EXT_PWR_DOWN_PHY_EN |
				 EXT_PWR_DOWN_PHY_RD |
				 EXT_PWR_DOWN_PHY_SD |
				 EXT_PWR_DOWN_PHY_RX |
				 EXT_PWR_DOWN_PHY_TX |
				 EXT_IDDQ_GLBL_PWR);
			reg |=   EXT_PHY_RESET;
			bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
			mdelay(1);

			reg &=  ~EXT_PHY_RESET;
		} else {
			reg &= ~EXT_PWR_DOWN_PHY;
			reg |= EXT_PWR_DN_EN_LD;
		}
		bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
		bcmgenet_phy_power_set(priv->dev, true);
		break;

	case GENET_POWER_CABLE_SENSE:
		/* enable APD */
		if (!GENET_IS_V5(priv)) {
			reg |= EXT_PWR_DN_EN_LD;
			bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
		}
		break;
	case GENET_POWER_WOL_MAGIC:
		bcmgenet_wol_power_up_cfg(priv, mode);
		return;
	default:
		break;
	}
}

/* ioctl handle special commands that are not present in ethtool. */
static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
{
	if (!netif_running(dev))
		return -EINVAL;

	if (!dev->phydev)
		return -ENODEV;

	return phy_mii_ioctl(dev->phydev, rq, cmd);
}

static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
					 struct bcmgenet_tx_ring *ring)
{
	struct enet_cb *tx_cb_ptr;

	tx_cb_ptr = ring->cbs;
	tx_cb_ptr += ring->write_ptr - ring->cb_ptr;

	/* Advancing local write pointer */
	if (ring->write_ptr == ring->end_ptr)
		ring->write_ptr = ring->cb_ptr;
	else
		ring->write_ptr++;

	return tx_cb_ptr;
}

static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv,
					 struct bcmgenet_tx_ring *ring)
{
	struct enet_cb *tx_cb_ptr;

	tx_cb_ptr = ring->cbs;
	tx_cb_ptr += ring->write_ptr - ring->cb_ptr;

	/* Rewinding local write pointer */
	if (ring->write_ptr == ring->cb_ptr)
		ring->write_ptr = ring->end_ptr;
	else
		ring->write_ptr--;

	return tx_cb_ptr;
}

static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)
{
	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
				 INTRL2_CPU_MASK_SET);
}

static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)
{
	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_RXDMA_DONE,
				 INTRL2_CPU_MASK_CLEAR);
}

static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
{
	bcmgenet_intrl2_1_writel(ring->priv,
				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
				 INTRL2_CPU_MASK_SET);
}

static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
{
	bcmgenet_intrl2_1_writel(ring->priv,
				 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
				 INTRL2_CPU_MASK_CLEAR);
}

static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)
{
	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
				 INTRL2_CPU_MASK_SET);
}

static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_tx_ring *ring)
{
	bcmgenet_intrl2_0_writel(ring->priv, UMAC_IRQ_TXDMA_DONE,
				 INTRL2_CPU_MASK_CLEAR);
}

static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)
{
	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
				 INTRL2_CPU_MASK_CLEAR);
}

static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)
{
	bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
				 INTRL2_CPU_MASK_SET);
}

/* Simple helper to free a transmit control block's resources
 * Returns an skb when the last transmit control block associated with the
 * skb is freed.  The skb should be freed by the caller if necessary.
 */
static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev,
					   struct enet_cb *cb)
{
	struct sk_buff *skb;

	skb = cb->skb;

	if (skb) {
		cb->skb = NULL;
		if (cb == GENET_CB(skb)->first_cb)
			dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
					 dma_unmap_len(cb, dma_len),
					 DMA_TO_DEVICE);
		else
			dma_unmap_page(dev, dma_unmap_addr(cb, dma_addr),
				       dma_unmap_len(cb, dma_len),
				       DMA_TO_DEVICE);
		dma_unmap_addr_set(cb, dma_addr, 0);

		if (cb == GENET_CB(skb)->last_cb)
			return skb;

	} else if (dma_unmap_addr(cb, dma_addr)) {
		dma_unmap_page(dev,
			       dma_unmap_addr(cb, dma_addr),
			       dma_unmap_len(cb, dma_len),
			       DMA_TO_DEVICE);
		dma_unmap_addr_set(cb, dma_addr, 0);
	}

	return NULL;
}

/* Simple helper to free a receive control block's resources */
static struct sk_buff *bcmgenet_free_rx_cb(struct device *dev,
					   struct enet_cb *cb)
{
	struct sk_buff *skb;

	skb = cb->skb;
	cb->skb = NULL;

	if (dma_unmap_addr(cb, dma_addr)) {
		dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
				 dma_unmap_len(cb, dma_len), DMA_FROM_DEVICE);
		dma_unmap_addr_set(cb, dma_addr, 0);
	}

	return skb;
}

/* Unlocked version of the reclaim routine */
static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
					  struct bcmgenet_tx_ring *ring)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	unsigned int txbds_processed = 0;
	unsigned int bytes_compl = 0;
	unsigned int pkts_compl = 0;
	unsigned int txbds_ready;
	unsigned int c_index;
	struct sk_buff *skb;

	/* Clear status before servicing to reduce spurious interrupts */
	if (ring->index == DESC_INDEX)
		bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_TXDMA_DONE,
					 INTRL2_CPU_CLEAR);
	else
		bcmgenet_intrl2_1_writel(priv, (1 << ring->index),
					 INTRL2_CPU_CLEAR);

	/* Compute how many buffers are transmitted since last xmit call */
	c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX)
		& DMA_C_INDEX_MASK;
	txbds_ready = (c_index - ring->c_index) & DMA_C_INDEX_MASK;

	netif_dbg(priv, tx_done, dev,
		  "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
		  __func__, ring->index, ring->c_index, c_index, txbds_ready);

	/* Reclaim transmitted buffers */
	while (txbds_processed < txbds_ready) {
		skb = bcmgenet_free_tx_cb(&priv->pdev->dev,
					  &priv->tx_cbs[ring->clean_ptr]);
		if (skb) {
			pkts_compl++;
			bytes_compl += GENET_CB(skb)->bytes_sent;
			dev_consume_skb_any(skb);
		}

		txbds_processed++;
		if (likely(ring->clean_ptr < ring->end_ptr))
			ring->clean_ptr++;
		else
			ring->clean_ptr = ring->cb_ptr;
	}

	ring->free_bds += txbds_processed;
	ring->c_index = c_index;

	ring->packets += pkts_compl;
	ring->bytes += bytes_compl;

	netdev_tx_completed_queue(netdev_get_tx_queue(dev, ring->queue),
				  pkts_compl, bytes_compl);

	return txbds_processed;
}

static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
				struct bcmgenet_tx_ring *ring)
{
	unsigned int released;

	spin_lock_bh(&ring->lock);
	released = __bcmgenet_tx_reclaim(dev, ring);
	spin_unlock_bh(&ring->lock);

	return released;
}

static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
{
	struct bcmgenet_tx_ring *ring =
		container_of(napi, struct bcmgenet_tx_ring, napi);
	unsigned int work_done = 0;
	struct netdev_queue *txq;

	spin_lock(&ring->lock);
	work_done = __bcmgenet_tx_reclaim(ring->priv->dev, ring);
	if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
		txq = netdev_get_tx_queue(ring->priv->dev, ring->queue);
		netif_tx_wake_queue(txq);
	}
	spin_unlock(&ring->lock);

	if (work_done == 0) {
		napi_complete(napi);
		ring->int_enable(ring);

		return 0;
	}

	return budget;
}

static void bcmgenet_tx_reclaim_all(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	int i;

	if (netif_is_multiqueue(dev)) {
		for (i = 0; i < priv->hw_params->tx_queues; i++)
			bcmgenet_tx_reclaim(dev, &priv->tx_rings[i]);
	}

	bcmgenet_tx_reclaim(dev, &priv->tx_rings[DESC_INDEX]);
}

/* Reallocate the SKB to put enough headroom in front of it and insert
 * the transmit checksum offsets in the descriptors
 */
static struct sk_buff *bcmgenet_put_tx_csum(struct net_device *dev,
					    struct sk_buff *skb)
{
	struct status_64 *status = NULL;
	struct sk_buff *new_skb;
	u16 offset;
	u8 ip_proto;
	__be16 ip_ver;
	u32 tx_csum_info;

	if (unlikely(skb_headroom(skb) < sizeof(*status))) {
		/* If 64 byte status block enabled, must make sure skb has
		 * enough headroom for us to insert 64B status block.
		 */
		new_skb = skb_realloc_headroom(skb, sizeof(*status));
		dev_kfree_skb(skb);
		if (!new_skb) {
			dev->stats.tx_dropped++;
			return NULL;
		}
		skb = new_skb;
	}

	skb_push(skb, sizeof(*status));
	status = (struct status_64 *)skb->data;

	if (skb->ip_summed  == CHECKSUM_PARTIAL) {
		ip_ver = skb->protocol;
		switch (ip_ver) {
		case htons(ETH_P_IP):
			ip_proto = ip_hdr(skb)->protocol;
			break;
		case htons(ETH_P_IPV6):
			ip_proto = ipv6_hdr(skb)->nexthdr;
			break;
		default:
			return skb;
		}

		offset = skb_checksum_start_offset(skb) - sizeof(*status);
		tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
				(offset + skb->csum_offset);

		/* Set the length valid bit for TCP and UDP and just set
		 * the special UDP flag for IPv4, else just set to 0.
		 */
		if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
			tx_csum_info |= STATUS_TX_CSUM_LV;
			if (ip_proto == IPPROTO_UDP &&
			    ip_ver == htons(ETH_P_IP))
				tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
		} else {
			tx_csum_info = 0;
		}

		status->tx_csum_info = tx_csum_info;
	}

	return skb;
}

static void bcmgenet_hide_tsb(struct sk_buff *skb)
{
	__skb_pull(skb, sizeof(struct status_64));
}

static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	struct device *kdev = &priv->pdev->dev;
	struct bcmgenet_tx_ring *ring = NULL;
	struct enet_cb *tx_cb_ptr;
	struct netdev_queue *txq;
	int nr_frags, index;
	dma_addr_t mapping;
	unsigned int size;
	skb_frag_t *frag;
	u32 len_stat;
	int ret;
	int i;

	index = skb_get_queue_mapping(skb);
	/* Mapping strategy:
	 * queue_mapping = 0, unclassified, packet xmited through ring16
	 * queue_mapping = 1, goes to ring 0. (highest priority queue
	 * queue_mapping = 2, goes to ring 1.
	 * queue_mapping = 3, goes to ring 2.
	 * queue_mapping = 4, goes to ring 3.
	 */
	if (index == 0)
		index = DESC_INDEX;
	else
		index -= 1;

	ring = &priv->tx_rings[index];
	txq = netdev_get_tx_queue(dev, ring->queue);

	nr_frags = skb_shinfo(skb)->nr_frags;

	spin_lock(&ring->lock);
	if (ring->free_bds <= (nr_frags + 1)) {
		if (!netif_tx_queue_stopped(txq)) {
			netif_tx_stop_queue(txq);
			netdev_err(dev,
				   "%s: tx ring %d full when queue %d awake\n",
				   __func__, index, ring->queue);
		}
		ret = NETDEV_TX_BUSY;
		goto out;
	}

	/* Retain how many bytes will be sent on the wire, without TSB inserted
	 * by transmit checksum offload
	 */
	GENET_CB(skb)->bytes_sent = skb->len;

	/* set the SKB transmit checksum */
	if (priv->desc_64b_en) {
		skb = bcmgenet_put_tx_csum(dev, skb);
		if (!skb) {
			ret = NETDEV_TX_OK;
			goto out;
		}
	}

	for (i = 0; i <= nr_frags; i++) {
		tx_cb_ptr = bcmgenet_get_txcb(priv, ring);

		BUG_ON(!tx_cb_ptr);

		if (!i) {
			/* Transmit single SKB or head of fragment list */
			GENET_CB(skb)->first_cb = tx_cb_ptr;
			size = skb_headlen(skb);
			mapping = dma_map_single(kdev, skb->data, size,
						 DMA_TO_DEVICE);
		} else {
			/* xmit fragment */
			frag = &skb_shinfo(skb)->frags[i - 1];
			size = skb_frag_size(frag);
			mapping = skb_frag_dma_map(kdev, frag, 0, size,
						   DMA_TO_DEVICE);
		}

		ret = dma_mapping_error(kdev, mapping);
		if (ret) {
			priv->mib.tx_dma_failed++;
			netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
			ret = NETDEV_TX_OK;
			goto out_unmap_frags;
		}
		dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
		dma_unmap_len_set(tx_cb_ptr, dma_len, size);

		tx_cb_ptr->skb = skb;

		len_stat = (size << DMA_BUFLENGTH_SHIFT) |
			   (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT);

		/* Note: if we ever change from DMA_TX_APPEND_CRC below we
		 * will need to restore software padding of "runt" packets
		 */
		len_stat |= DMA_TX_APPEND_CRC;

		if (!i) {
			len_stat |= DMA_SOP;
			if (skb->ip_summed == CHECKSUM_PARTIAL)
				len_stat |= DMA_TX_DO_CSUM;
		}
		if (i == nr_frags)
			len_stat |= DMA_EOP;

		dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, len_stat);
	}

	GENET_CB(skb)->last_cb = tx_cb_ptr;

	bcmgenet_hide_tsb(skb);
	skb_tx_timestamp(skb);

	/* Decrement total BD count and advance our write pointer */
	ring->free_bds -= nr_frags + 1;
	ring->prod_index += nr_frags + 1;
	ring->prod_index &= DMA_P_INDEX_MASK;

	netdev_tx_sent_queue(txq, GENET_CB(skb)->bytes_sent);

	if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
		netif_tx_stop_queue(txq);

	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
		/* Packets are ready, update producer index */
		bcmgenet_tdma_ring_writel(priv, ring->index,
					  ring->prod_index, TDMA_PROD_INDEX);
out:
	spin_unlock(&ring->lock);

	return ret;

out_unmap_frags:
	/* Back up for failed control block mapping */
	bcmgenet_put_txcb(priv, ring);

	/* Unmap successfully mapped control blocks */
	while (i-- > 0) {
		tx_cb_ptr = bcmgenet_put_txcb(priv, ring);
		bcmgenet_free_tx_cb(kdev, tx_cb_ptr);
	}

	dev_kfree_skb(skb);
	goto out;
}

static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
					  struct enet_cb *cb)
{
	struct device *kdev = &priv->pdev->dev;
	struct sk_buff *skb;
	struct sk_buff *rx_skb;
	dma_addr_t mapping;

	/* Allocate a new Rx skb */
	skb = __netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT,
				 GFP_ATOMIC | __GFP_NOWARN);
	if (!skb) {
		priv->mib.alloc_rx_buff_failed++;
		netif_err(priv, rx_err, priv->dev,
			  "%s: Rx skb allocation failed\n", __func__);
		return NULL;
	}

	/* DMA-map the new Rx skb */
	mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len,
				 DMA_FROM_DEVICE);
	if (dma_mapping_error(kdev, mapping)) {
		priv->mib.rx_dma_failed++;
		dev_kfree_skb_any(skb);
		netif_err(priv, rx_err, priv->dev,
			  "%s: Rx skb DMA mapping failed\n", __func__);
		return NULL;
	}

	/* Grab the current Rx skb from the ring and DMA-unmap it */
	rx_skb = bcmgenet_free_rx_cb(kdev, cb);

	/* Put the new Rx skb on the ring */
	cb->skb = skb;
	dma_unmap_addr_set(cb, dma_addr, mapping);
	dma_unmap_len_set(cb, dma_len, priv->rx_buf_len);
	dmadesc_set_addr(priv, cb->bd_addr, mapping);

	/* Return the current Rx skb to caller */
	return rx_skb;
}

/* bcmgenet_desc_rx - descriptor based rx process.
 * this could be called from bottom half, or from NAPI polling method.
 */
static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
				     unsigned int budget)
{
	struct bcmgenet_priv *priv = ring->priv;
	struct net_device *dev = priv->dev;
	struct enet_cb *cb;
	struct sk_buff *skb;
	u32 dma_length_status;
	unsigned long dma_flag;
	int len;
	unsigned int rxpktprocessed = 0, rxpkttoprocess;
	unsigned int bytes_processed = 0;
	unsigned int p_index, mask;
	unsigned int discards;
	unsigned int chksum_ok = 0;

	/* Clear status before servicing to reduce spurious interrupts */
	if (ring->index == DESC_INDEX) {
		bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_DONE,
					 INTRL2_CPU_CLEAR);
	} else {
		mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index);
		bcmgenet_intrl2_1_writel(priv,
					 mask,
					 INTRL2_CPU_CLEAR);
	}

	p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);

	discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
		   DMA_P_INDEX_DISCARD_CNT_MASK;
	if (discards > ring->old_discards) {
		discards = discards - ring->old_discards;
		ring->errors += discards;
		ring->old_discards += discards;

		/* Clear HW register when we reach 75% of maximum 0xFFFF */
		if (ring->old_discards >= 0xC000) {
			ring->old_discards = 0;
			bcmgenet_rdma_ring_writel(priv, ring->index, 0,
						  RDMA_PROD_INDEX);
		}
	}

	p_index &= DMA_P_INDEX_MASK;
	rxpkttoprocess = (p_index - ring->c_index) & DMA_C_INDEX_MASK;

	netif_dbg(priv, rx_status, dev,
		  "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);

	while ((rxpktprocessed < rxpkttoprocess) &&
	       (rxpktprocessed < budget)) {
		cb = &priv->rx_cbs[ring->read_ptr];
		skb = bcmgenet_rx_refill(priv, cb);

		if (unlikely(!skb)) {
			ring->dropped++;
			goto next;
		}

		if (!priv->desc_64b_en) {
			dma_length_status =
				dmadesc_get_length_status(priv, cb->bd_addr);
		} else {
			struct status_64 *status;

			status = (struct status_64 *)skb->data;
			dma_length_status = status->length_status;
		}

		/* DMA flags and length are still valid no matter how
		 * we got the Receive Status Vector (64B RSB or register)
		 */
		dma_flag = dma_length_status & 0xffff;
		len = dma_length_status >> DMA_BUFLENGTH_SHIFT;

		netif_dbg(priv, rx_status, dev,
			  "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
			  __func__, p_index, ring->c_index,
			  ring->read_ptr, dma_length_status);

		if (unlikely(len > RX_BUF_LENGTH)) {
			netif_err(priv, rx_status, dev, "oversized packet\n");
			dev->stats.rx_length_errors++;
			dev->stats.rx_errors++;
			dev_kfree_skb_any(skb);
			goto next;
		}

		if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
			netif_err(priv, rx_status, dev,
				  "dropping fragmented packet!\n");
			ring->errors++;
			dev_kfree_skb_any(skb);
			goto next;
		}

		/* report errors */
		if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
						DMA_RX_OV |
						DMA_RX_NO |
						DMA_RX_LG |
						DMA_RX_RXER))) {
			netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
				  (unsigned int)dma_flag);
			if (dma_flag & DMA_RX_CRC_ERROR)
				dev->stats.rx_crc_errors++;
			if (dma_flag & DMA_RX_OV)
				dev->stats.rx_over_errors++;
			if (dma_flag & DMA_RX_NO)
				dev->stats.rx_frame_errors++;
			if (dma_flag & DMA_RX_LG)
				dev->stats.rx_length_errors++;
			dev->stats.rx_errors++;
			dev_kfree_skb_any(skb);
			goto next;
		} /* error packet */

		chksum_ok = (dma_flag & priv->dma_rx_chk_bit) &&
			     priv->desc_rxchk_en;

		skb_put(skb, len);
		if (priv->desc_64b_en) {
			skb_pull(skb, 64);
			len -= 64;
		}

		if (likely(chksum_ok))
			skb->ip_summed = CHECKSUM_UNNECESSARY;

		/* remove hardware 2bytes added for IP alignment */
		skb_pull(skb, 2);
		len -= 2;

		if (priv->crc_fwd_en) {
			skb_trim(skb, len - ETH_FCS_LEN);
			len -= ETH_FCS_LEN;
		}

		bytes_processed += len;

		/*Finish setting up the received SKB and send it to the kernel*/
		skb->protocol = eth_type_trans(skb, priv->dev);
		ring->packets++;
		ring->bytes += len;
		if (dma_flag & DMA_RX_MULT)
			dev->stats.multicast++;

		/* Notify kernel */
		napi_gro_receive(&ring->napi, skb);
		netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");

next:
		rxpktprocessed++;
		if (likely(ring->read_ptr < ring->end_ptr))
			ring->read_ptr++;
		else
			ring->read_ptr = ring->cb_ptr;

		ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
		bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
	}

	ring->dim.bytes = bytes_processed;
	ring->dim.packets = rxpktprocessed;

	return rxpktprocessed;
}

/* Rx NAPI polling method */
static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
{
	struct bcmgenet_rx_ring *ring = container_of(napi,
			struct bcmgenet_rx_ring, napi);
	struct dim_sample dim_sample = {};
	unsigned int work_done;

	work_done = bcmgenet_desc_rx(ring, budget);

	if (work_done < budget) {
		napi_complete_done(napi, work_done);
		ring->int_enable(ring);
	}

	if (ring->dim.use_dim) {
		dim_update_sample(ring->dim.event_ctr, ring->dim.packets,
				  ring->dim.bytes, &dim_sample);
		net_dim(&ring->dim.dim, dim_sample);
	}

	return work_done;
}

static void bcmgenet_dim_work(struct work_struct *work)
{
	struct dim *dim = container_of(work, struct dim, work);
	struct bcmgenet_net_dim *ndim =
			container_of(dim, struct bcmgenet_net_dim, dim);
	struct bcmgenet_rx_ring *ring =
			container_of(ndim, struct bcmgenet_rx_ring, dim);
	struct dim_cq_moder cur_profile =
			net_dim_get_rx_moderation(dim->mode, dim->profile_ix);

	bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts);
	dim->state = DIM_START_MEASURE;
}

/* Assign skb to RX DMA descriptor. */
static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
				     struct bcmgenet_rx_ring *ring)
{
	struct enet_cb *cb;
	struct sk_buff *skb;
	int i;

	netif_dbg(priv, hw, priv->dev, "%s\n", __func__);

	/* loop here for each buffer needing assign */
	for (i = 0; i < ring->size; i++) {
		cb = ring->cbs + i;
		skb = bcmgenet_rx_refill(priv, cb);
		if (skb)
			dev_consume_skb_any(skb);
		if (!cb->skb)
			return -ENOMEM;
	}

	return 0;
}

static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
{
	struct sk_buff *skb;
	struct enet_cb *cb;
	int i;

	for (i = 0; i < priv->num_rx_bds; i++) {
		cb = &priv->rx_cbs[i];

		skb = bcmgenet_free_rx_cb(&priv->pdev->dev, cb);
		if (skb)
			dev_consume_skb_any(skb);
	}
}

static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
{
	u32 reg;

	reg = bcmgenet_umac_readl(priv, UMAC_CMD);
	if (enable)
		reg |= mask;
	else
		reg &= ~mask;
	bcmgenet_umac_writel(priv, reg, UMAC_CMD);

	/* UniMAC stops on a packet boundary, wait for a full-size packet
	 * to be processed
	 */
	if (enable == 0)
		usleep_range(1000, 2000);
}

static void reset_umac(struct bcmgenet_priv *priv)
{
	/* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
	bcmgenet_rbuf_ctrl_set(priv, 0);
	udelay(10);

	/* disable MAC while updating its registers */
	bcmgenet_umac_writel(priv, 0, UMAC_CMD);

	/* issue soft reset with (rg)mii loopback to ensure a stable rxclk */
	bcmgenet_umac_writel(priv, CMD_SW_RESET | CMD_LCL_LOOP_EN, UMAC_CMD);
}

static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
{
	/* Mask all interrupts.*/
	bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
	bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
	bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
	bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
}

static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv)
{
	u32 int0_enable = 0;

	/* Monitor cable plug/unplugged event for internal PHY, external PHY
	 * and MoCA PHY
	 */
	if (priv->internal_phy) {
		int0_enable |= UMAC_IRQ_LINK_EVENT;
		if (GENET_IS_V1(priv) || GENET_IS_V2(priv) || GENET_IS_V3(priv))
			int0_enable |= UMAC_IRQ_PHY_DET_R;
	} else if (priv->ext_phy) {
		int0_enable |= UMAC_IRQ_LINK_EVENT;
	} else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
		if (priv->hw_params->flags & GENET_HAS_MOCA_LINK_DET)
			int0_enable |= UMAC_IRQ_LINK_EVENT;
	}
	bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
}

static void init_umac(struct bcmgenet_priv *priv)
{
	struct device *kdev = &priv->pdev->dev;
	u32 reg;
	u32 int0_enable = 0;

	dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");

	reset_umac(priv);

	/* clear tx/rx counter */
	bcmgenet_umac_writel(priv,
			     MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
			     UMAC_MIB_CTRL);
	bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);

	bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);

	/* init rx registers, enable ip header optimization */
	reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
	reg |= RBUF_ALIGN_2B;
	bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);

	if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
		bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);

	bcmgenet_intr_disable(priv);

	/* Configure backpressure vectors for MoCA */
	if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
		reg = bcmgenet_bp_mc_get(priv);
		reg |= BIT(priv->hw_params->bp_in_en_shift);

		/* bp_mask: back pressure mask */
		if (netif_is_multiqueue(priv->dev))
			reg |= priv->hw_params->bp_in_mask;
		else
			reg &= ~priv->hw_params->bp_in_mask;
		bcmgenet_bp_mc_set(priv, reg);
	}

	/* Enable MDIO interrupts on GENET v3+ */
	if (priv->hw_params->flags & GENET_HAS_MDIO_INTR)
		int0_enable |= (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);

	bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);

	dev_dbg(kdev, "done init umac\n");
}

static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring,
			      void (*cb)(struct work_struct *work))
{
	struct bcmgenet_net_dim *dim = &ring->dim;

	INIT_WORK(&dim->dim.work, cb);
	dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
	dim->event_ctr = 0;
	dim->packets = 0;
	dim->bytes = 0;
}

static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring)
{
	struct bcmgenet_net_dim *dim = &ring->dim;
	struct dim_cq_moder moder;
	u32 usecs, pkts;

	usecs = ring->rx_coalesce_usecs;
	pkts = ring->rx_max_coalesced_frames;

	/* If DIM was enabled, re-apply default parameters */
	if (dim->use_dim) {
		moder = net_dim_get_def_rx_moderation(dim->dim.mode);
		usecs = moder.usec;
		pkts = moder.pkts;
	}

	bcmgenet_set_rx_coalesce(ring, usecs, pkts);
}

/* Initialize a Tx ring along with corresponding hardware registers */
static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
				  unsigned int index, unsigned int size,
				  unsigned int start_ptr, unsigned int end_ptr)
{
	struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
	u32 words_per_bd = WORDS_PER_BD(priv);
	u32 flow_period_val = 0;

	spin_lock_init(&ring->lock);
	ring->priv = priv;
	ring->index = index;
	if (index == DESC_INDEX) {
		ring->queue = 0;
		ring->int_enable = bcmgenet_tx_ring16_int_enable;
		ring->int_disable = bcmgenet_tx_ring16_int_disable;
	} else {
		ring->queue = index + 1;
		ring->int_enable = bcmgenet_tx_ring_int_enable;
		ring->int_disable = bcmgenet_tx_ring_int_disable;
	}
	ring->cbs = priv->tx_cbs + start_ptr;
	ring->size = size;
	ring->clean_ptr = start_ptr;
	ring->c_index = 0;
	ring->free_bds = size;
	ring->write_ptr = start_ptr;
	ring->cb_ptr = start_ptr;
	ring->end_ptr = end_ptr - 1;
	ring->prod_index = 0;

	/* Set flow period for ring != 16 */
	if (index != DESC_INDEX)
		flow_period_val = ENET_MAX_MTU_SIZE << 16;

	bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
	bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
	bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
	/* Disable rate control for now */
	bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
				  TDMA_FLOW_PERIOD);
	bcmgenet_tdma_ring_writel(priv, index,
				  ((size << DMA_RING_SIZE_SHIFT) |
				   RX_BUF_LENGTH), DMA_RING_BUF_SIZE);

	/* Set start and end address, read and write pointers */
	bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  DMA_START_ADDR);
	bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  TDMA_READ_PTR);
	bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  TDMA_WRITE_PTR);
	bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
				  DMA_END_ADDR);

	/* Initialize Tx NAPI */
	netif_tx_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll,
			  NAPI_POLL_WEIGHT);
}

/* Initialize a RDMA ring */
static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
				 unsigned int index, unsigned int size,
				 unsigned int start_ptr, unsigned int end_ptr)
{
	struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
	u32 words_per_bd = WORDS_PER_BD(priv);
	int ret;

	ring->priv = priv;
	ring->index = index;
	if (index == DESC_INDEX) {
		ring->int_enable = bcmgenet_rx_ring16_int_enable;
		ring->int_disable = bcmgenet_rx_ring16_int_disable;
	} else {
		ring->int_enable = bcmgenet_rx_ring_int_enable;
		ring->int_disable = bcmgenet_rx_ring_int_disable;
	}
	ring->cbs = priv->rx_cbs + start_ptr;
	ring->size = size;
	ring->c_index = 0;
	ring->read_ptr = start_ptr;
	ring->cb_ptr = start_ptr;
	ring->end_ptr = end_ptr - 1;

	ret = bcmgenet_alloc_rx_buffers(priv, ring);
	if (ret)
		return ret;

	bcmgenet_init_dim(ring, bcmgenet_dim_work);
	bcmgenet_init_rx_coalesce(ring);

	/* Initialize Rx NAPI */
	netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll,
		       NAPI_POLL_WEIGHT);

	bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
	bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
	bcmgenet_rdma_ring_writel(priv, index,
				  ((size << DMA_RING_SIZE_SHIFT) |
				   RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
	bcmgenet_rdma_ring_writel(priv, index,
				  (DMA_FC_THRESH_LO <<
				   DMA_XOFF_THRESHOLD_SHIFT) |
				   DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);

	/* Set start and end address, read and write pointers */
	bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  DMA_START_ADDR);
	bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  RDMA_READ_PTR);
	bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
				  RDMA_WRITE_PTR);
	bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
				  DMA_END_ADDR);

	return ret;
}

static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_tx_ring *ring;

	for (i = 0; i < priv->hw_params->tx_queues; ++i) {
		ring = &priv->tx_rings[i];
		napi_enable(&ring->napi);
		ring->int_enable(ring);
	}

	ring = &priv->tx_rings[DESC_INDEX];
	napi_enable(&ring->napi);
	ring->int_enable(ring);
}

static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_tx_ring *ring;

	for (i = 0; i < priv->hw_params->tx_queues; ++i) {
		ring = &priv->tx_rings[i];
		napi_disable(&ring->napi);
	}

	ring = &priv->tx_rings[DESC_INDEX];
	napi_disable(&ring->napi);
}

static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_tx_ring *ring;

	for (i = 0; i < priv->hw_params->tx_queues; ++i) {
		ring = &priv->tx_rings[i];
		netif_napi_del(&ring->napi);
	}

	ring = &priv->tx_rings[DESC_INDEX];
	netif_napi_del(&ring->napi);
}

/* Initialize Tx queues
 *
 * Queues 0-3 are priority-based, each one has 32 descriptors,
 * with queue 0 being the highest priority queue.
 *
 * Queue 16 is the default Tx queue with
 * GENET_Q16_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
 *
 * The transmit control block pool is then partitioned as follows:
 * - Tx queue 0 uses tx_cbs[0..31]
 * - Tx queue 1 uses tx_cbs[32..63]
 * - Tx queue 2 uses tx_cbs[64..95]
 * - Tx queue 3 uses tx_cbs[96..127]
 * - Tx queue 16 uses tx_cbs[128..255]
 */
static void bcmgenet_init_tx_queues(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	u32 i, dma_enable;
	u32 dma_ctrl, ring_cfg;
	u32 dma_priority[3] = {0, 0, 0};

	dma_ctrl = bcmgenet_tdma_readl(priv, DMA_CTRL);
	dma_enable = dma_ctrl & DMA_EN;
	dma_ctrl &= ~DMA_EN;
	bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);

	dma_ctrl = 0;
	ring_cfg = 0;

	/* Enable strict priority arbiter mode */
	bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL);

	/* Initialize Tx priority queues */
	for (i = 0; i < priv->hw_params->tx_queues; i++) {
		bcmgenet_init_tx_ring(priv, i, priv->hw_params->tx_bds_per_q,
				      i * priv->hw_params->tx_bds_per_q,
				      (i + 1) * priv->hw_params->tx_bds_per_q);
		ring_cfg |= (1 << i);
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
		dma_priority[DMA_PRIO_REG_INDEX(i)] |=
			((GENET_Q0_PRIORITY + i) << DMA_PRIO_REG_SHIFT(i));
	}

	/* Initialize Tx default queue 16 */
	bcmgenet_init_tx_ring(priv, DESC_INDEX, GENET_Q16_TX_BD_CNT,
			      priv->hw_params->tx_queues *
			      priv->hw_params->tx_bds_per_q,
			      TOTAL_DESC);
	ring_cfg |= (1 << DESC_INDEX);
	dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
	dma_priority[DMA_PRIO_REG_INDEX(DESC_INDEX)] |=
		((GENET_Q0_PRIORITY + priv->hw_params->tx_queues) <<
		 DMA_PRIO_REG_SHIFT(DESC_INDEX));

	/* Set Tx queue priorities */
	bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
	bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
	bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);

	/* Enable Tx queues */
	bcmgenet_tdma_writel(priv, ring_cfg, DMA_RING_CFG);

	/* Enable Tx DMA */
	if (dma_enable)
		dma_ctrl |= DMA_EN;
	bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
}

static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_rx_ring *ring;

	for (i = 0; i < priv->hw_params->rx_queues; ++i) {
		ring = &priv->rx_rings[i];
		napi_enable(&ring->napi);
		ring->int_enable(ring);
	}

	ring = &priv->rx_rings[DESC_INDEX];
	napi_enable(&ring->napi);
	ring->int_enable(ring);
}

static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_rx_ring *ring;

	for (i = 0; i < priv->hw_params->rx_queues; ++i) {
		ring = &priv->rx_rings[i];
		napi_disable(&ring->napi);
		cancel_work_sync(&ring->dim.dim.work);
	}

	ring = &priv->rx_rings[DESC_INDEX];
	napi_disable(&ring->napi);
	cancel_work_sync(&ring->dim.dim.work);
}

static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
{
	unsigned int i;
	struct bcmgenet_rx_ring *ring;

	for (i = 0; i < priv->hw_params->rx_queues; ++i) {
		ring = &priv->rx_rings[i];
		netif_napi_del(&ring->napi);
	}

	ring = &priv->rx_rings[DESC_INDEX];
	netif_napi_del(&ring->napi);
}

/* Initialize Rx queues
 *
 * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
 * used to direct traffic to these queues.
 *
 * Queue 16 is the default Rx queue with GENET_Q16_RX_BD_CNT descriptors.
 */
static int bcmgenet_init_rx_queues(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	u32 i;
	u32 dma_enable;
	u32 dma_ctrl;
	u32 ring_cfg;
	int ret;

	dma_ctrl = bcmgenet_rdma_readl(priv, DMA_CTRL);
	dma_enable = dma_ctrl & DMA_EN;
	dma_ctrl &= ~DMA_EN;
	bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);

	dma_ctrl = 0;
	ring_cfg = 0;

	/* Initialize Rx priority queues */
	for (i = 0; i < priv->hw_params->rx_queues; i++) {
		ret = bcmgenet_init_rx_ring(priv, i,
					    priv->hw_params->rx_bds_per_q,
					    i * priv->hw_params->rx_bds_per_q,
					    (i + 1) *
					    priv->hw_params->rx_bds_per_q);
		if (ret)
			return ret;

		ring_cfg |= (1 << i);
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
	}

	/* Initialize Rx default queue 16 */
	ret = bcmgenet_init_rx_ring(priv, DESC_INDEX, GENET_Q16_RX_BD_CNT,
				    priv->hw_params->rx_queues *
				    priv->hw_params->rx_bds_per_q,
				    TOTAL_DESC);
	if (ret)
		return ret;

	ring_cfg |= (1 << DESC_INDEX);
	dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));

	/* Enable rings */
	bcmgenet_rdma_writel(priv, ring_cfg, DMA_RING_CFG);

	/* Configure ring as descriptor ring and re-enable DMA if enabled */
	if (dma_enable)
		dma_ctrl |= DMA_EN;
	bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);

	return 0;
}

static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
{
	int ret = 0;
	int timeout = 0;
	u32 reg;
	u32 dma_ctrl;
	int i;

	/* Disable TDMA to stop add more frames in TX DMA */
	reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
	reg &= ~DMA_EN;
	bcmgenet_tdma_writel(priv, reg, DMA_CTRL);

	/* Check TDMA status register to confirm TDMA is disabled */
	while (timeout++ < DMA_TIMEOUT_VAL) {
		reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
		if (reg & DMA_DISABLED)
			break;

		udelay(1);
	}

	if (timeout == DMA_TIMEOUT_VAL) {
		netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
		ret = -ETIMEDOUT;
	}

	/* Wait 10ms for packet drain in both tx and rx dma */
	usleep_range(10000, 20000);

	/* Disable RDMA */
	reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
	reg &= ~DMA_EN;
	bcmgenet_rdma_writel(priv, reg, DMA_CTRL);

	timeout = 0;
	/* Check RDMA status register to confirm RDMA is disabled */
	while (timeout++ < DMA_TIMEOUT_VAL) {
		reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
		if (reg & DMA_DISABLED)
			break;

		udelay(1);
	}

	if (timeout == DMA_TIMEOUT_VAL) {
		netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
		ret = -ETIMEDOUT;
	}

	dma_ctrl = 0;
	for (i = 0; i < priv->hw_params->rx_queues; i++)
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
	reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
	reg &= ~dma_ctrl;
	bcmgenet_rdma_writel(priv, reg, DMA_CTRL);

	dma_ctrl = 0;
	for (i = 0; i < priv->hw_params->tx_queues; i++)
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
	reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
	reg &= ~dma_ctrl;
	bcmgenet_tdma_writel(priv, reg, DMA_CTRL);

	return ret;
}

static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
{
	struct netdev_queue *txq;
	int i;

	bcmgenet_fini_rx_napi(priv);
	bcmgenet_fini_tx_napi(priv);

	for (i = 0; i < priv->num_tx_bds; i++)
		dev_kfree_skb(bcmgenet_free_tx_cb(&priv->pdev->dev,
						  priv->tx_cbs + i));

	for (i = 0; i < priv->hw_params->tx_queues; i++) {
		txq = netdev_get_tx_queue(priv->dev, priv->tx_rings[i].queue);
		netdev_tx_reset_queue(txq);
	}

	txq = netdev_get_tx_queue(priv->dev, priv->tx_rings[DESC_INDEX].queue);
	netdev_tx_reset_queue(txq);

	bcmgenet_free_rx_buffers(priv);
	kfree(priv->rx_cbs);
	kfree(priv->tx_cbs);
}

/* init_edma: Initialize DMA control register */
static int bcmgenet_init_dma(struct bcmgenet_priv *priv)
{
	int ret;
	unsigned int i;
	struct enet_cb *cb;

	netif_dbg(priv, hw, priv->dev, "%s\n", __func__);

	/* Initialize common Rx ring structures */
	priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
	priv->num_rx_bds = TOTAL_DESC;
	priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct enet_cb),
			       GFP_KERNEL);
	if (!priv->rx_cbs)
		return -ENOMEM;

	for (i = 0; i < priv->num_rx_bds; i++) {
		cb = priv->rx_cbs + i;
		cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
	}

	/* Initialize common TX ring structures */
	priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
	priv->num_tx_bds = TOTAL_DESC;
	priv->tx_cbs = kcalloc(priv->num_tx_bds, sizeof(struct enet_cb),
			       GFP_KERNEL);
	if (!priv->tx_cbs) {
		kfree(priv->rx_cbs);
		return -ENOMEM;
	}

	for (i = 0; i < priv->num_tx_bds; i++) {
		cb = priv->tx_cbs + i;
		cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
	}

	/* Init rDma */
	bcmgenet_rdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);

	/* Initialize Rx queues */
	ret = bcmgenet_init_rx_queues(priv->dev);
	if (ret) {
		netdev_err(priv->dev, "failed to initialize Rx queues\n");
		bcmgenet_free_rx_buffers(priv);
		kfree(priv->rx_cbs);
		kfree(priv->tx_cbs);
		return ret;
	}

	/* Init tDma */
	bcmgenet_tdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);

	/* Initialize Tx queues */
	bcmgenet_init_tx_queues(priv->dev);

	return 0;
}

/* Interrupt bottom half */
static void bcmgenet_irq_task(struct work_struct *work)
{
	unsigned int status;
	struct bcmgenet_priv *priv = container_of(
			work, struct bcmgenet_priv, bcmgenet_irq_work);

	netif_dbg(priv, intr, priv->dev, "%s\n", __func__);

	spin_lock_irq(&priv->lock);
	status = priv->irq0_stat;
	priv->irq0_stat = 0;
	spin_unlock_irq(&priv->lock);

	if (status & UMAC_IRQ_PHY_DET_R &&
	    priv->dev->phydev->autoneg != AUTONEG_ENABLE) {
		phy_init_hw(priv->dev->phydev);
		genphy_config_aneg(priv->dev->phydev);
	}

	/* Link UP/DOWN event */
	if (status & UMAC_IRQ_LINK_EVENT)
		phy_mac_interrupt(priv->dev->phydev);

}

/* bcmgenet_isr1: handle Rx and Tx priority queues */
static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
{
	struct bcmgenet_priv *priv = dev_id;
	struct bcmgenet_rx_ring *rx_ring;
	struct bcmgenet_tx_ring *tx_ring;
	unsigned int index, status;

	/* Read irq status */
	status = bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
		~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);

	/* clear interrupts */
	bcmgenet_intrl2_1_writel(priv, status, INTRL2_CPU_CLEAR);

	netif_dbg(priv, intr, priv->dev,
		  "%s: IRQ=0x%x\n", __func__, status);

	/* Check Rx priority queue interrupts */
	for (index = 0; index < priv->hw_params->rx_queues; index++) {
		if (!(status & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
			continue;

		rx_ring = &priv->rx_rings[index];
		rx_ring->dim.event_ctr++;

		if (likely(napi_schedule_prep(&rx_ring->napi))) {
			rx_ring->int_disable(rx_ring);
			__napi_schedule_irqoff(&rx_ring->napi);
		}
	}

	/* Check Tx priority queue interrupts */
	for (index = 0; index < priv->hw_params->tx_queues; index++) {
		if (!(status & BIT(index)))
			continue;

		tx_ring = &priv->tx_rings[index];

		if (likely(napi_schedule_prep(&tx_ring->napi))) {
			tx_ring->int_disable(tx_ring);
			__napi_schedule_irqoff(&tx_ring->napi);
		}
	}

	return IRQ_HANDLED;
}

/* bcmgenet_isr0: handle Rx and Tx default queues + other stuff */
static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
{
	struct bcmgenet_priv *priv = dev_id;
	struct bcmgenet_rx_ring *rx_ring;
	struct bcmgenet_tx_ring *tx_ring;
	unsigned int status;
	unsigned long flags;

	/* Read irq status */
	status = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
		~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);

	/* clear interrupts */
	bcmgenet_intrl2_0_writel(priv, status, INTRL2_CPU_CLEAR);

	netif_dbg(priv, intr, priv->dev,
		  "IRQ=0x%x\n", status);

	if (status & UMAC_IRQ_RXDMA_DONE) {
		rx_ring = &priv->rx_rings[DESC_INDEX];
		rx_ring->dim.event_ctr++;

		if (likely(napi_schedule_prep(&rx_ring->napi))) {
			rx_ring->int_disable(rx_ring);
			__napi_schedule_irqoff(&rx_ring->napi);
		}
	}

	if (status & UMAC_IRQ_TXDMA_DONE) {
		tx_ring = &priv->tx_rings[DESC_INDEX];

		if (likely(napi_schedule_prep(&tx_ring->napi))) {
			tx_ring->int_disable(tx_ring);
			__napi_schedule_irqoff(&tx_ring->napi);
		}
	}

	if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
		status & (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR)) {
		wake_up(&priv->wq);
	}

	/* all other interested interrupts handled in bottom half */
	status &= (UMAC_IRQ_LINK_EVENT | UMAC_IRQ_PHY_DET_R);
	if (status) {
		/* Save irq status for bottom-half processing. */
		spin_lock_irqsave(&priv->lock, flags);
		priv->irq0_stat |= status;
		spin_unlock_irqrestore(&priv->lock, flags);

		schedule_work(&priv->bcmgenet_irq_work);
	}

	return IRQ_HANDLED;
}

static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
{
	struct bcmgenet_priv *priv = dev_id;

	pm_wakeup_event(&priv->pdev->dev, 0);

	return IRQ_HANDLED;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static void bcmgenet_poll_controller(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	/* Invoke the main RX/TX interrupt handler */
	disable_irq(priv->irq0);
	bcmgenet_isr0(priv->irq0, priv);
	enable_irq(priv->irq0);

	/* And the interrupt handler for RX/TX priority queues */
	disable_irq(priv->irq1);
	bcmgenet_isr1(priv->irq1, priv);
	enable_irq(priv->irq1);
}
#endif

static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
{
	u32 reg;

	reg = bcmgenet_rbuf_ctrl_get(priv);
	reg |= BIT(1);
	bcmgenet_rbuf_ctrl_set(priv, reg);
	udelay(10);

	reg &= ~BIT(1);
	bcmgenet_rbuf_ctrl_set(priv, reg);
	udelay(10);
}

static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
				 unsigned char *addr)
{
	bcmgenet_umac_writel(priv, (addr[0] << 24) | (addr[1] << 16) |
			(addr[2] << 8) | addr[3], UMAC_MAC0);
	bcmgenet_umac_writel(priv, (addr[4] << 8) | addr[5], UMAC_MAC1);
}

/* Returns a reusable dma control register value */
static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv)
{
	unsigned int i;
	u32 reg;
	u32 dma_ctrl;

	/* disable DMA */
	dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN;
	for (i = 0; i < priv->hw_params->tx_queues; i++)
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
	reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
	reg &= ~dma_ctrl;
	bcmgenet_tdma_writel(priv, reg, DMA_CTRL);

	dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN;
	for (i = 0; i < priv->hw_params->rx_queues; i++)
		dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
	reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
	reg &= ~dma_ctrl;
	bcmgenet_rdma_writel(priv, reg, DMA_CTRL);

	bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
	udelay(10);
	bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);

	return dma_ctrl;
}

static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl)
{
	u32 reg;

	reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
	reg |= dma_ctrl;
	bcmgenet_rdma_writel(priv, reg, DMA_CTRL);

	reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
	reg |= dma_ctrl;
	bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
}

/* bcmgenet_hfb_clear
 *
 * Clear Hardware Filter Block and disable all filtering.
 */
static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv)
{
	u32 i;

	bcmgenet_hfb_reg_writel(priv, 0x0, HFB_CTRL);
	bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS);
	bcmgenet_hfb_reg_writel(priv, 0x0, HFB_FLT_ENABLE_V3PLUS + 4);

	for (i = DMA_INDEX2RING_0; i <= DMA_INDEX2RING_7; i++)
		bcmgenet_rdma_writel(priv, 0x0, i);

	for (i = 0; i < (priv->hw_params->hfb_filter_cnt / 4); i++)
		bcmgenet_hfb_reg_writel(priv, 0x0,
					HFB_FLT_LEN_V3PLUS + i * sizeof(u32));

	for (i = 0; i < priv->hw_params->hfb_filter_cnt *
			priv->hw_params->hfb_filter_size; i++)
		bcmgenet_hfb_writel(priv, 0x0, i * sizeof(u32));
}

static void bcmgenet_hfb_init(struct bcmgenet_priv *priv)
{
	if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
		return;

	bcmgenet_hfb_clear(priv);
}

static void bcmgenet_netif_start(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	/* Start the network engine */
	bcmgenet_set_rx_mode(dev);
	bcmgenet_enable_rx_napi(priv);

	umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);

	bcmgenet_enable_tx_napi(priv);

	/* Monitor link interrupts now */
	bcmgenet_link_intr_enable(priv);

	phy_start(dev->phydev);
}

static int bcmgenet_open(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	unsigned long dma_ctrl;
	u32 reg;
	int ret;

	netif_dbg(priv, ifup, dev, "bcmgenet_open\n");

	/* Turn on the clock */
	clk_prepare_enable(priv->clk);

	/* If this is an internal GPHY, power it back on now, before UniMAC is
	 * brought out of reset as absolutely no UniMAC activity is allowed
	 */
	if (priv->internal_phy)
		bcmgenet_power_up(priv, GENET_POWER_PASSIVE);

	/* take MAC out of reset */
	bcmgenet_umac_reset(priv);

	init_umac(priv);

	/* Make sure we reflect the value of CRC_CMD_FWD */
	reg = bcmgenet_umac_readl(priv, UMAC_CMD);
	priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);

	bcmgenet_set_hw_addr(priv, dev->dev_addr);

	/* Disable RX/TX DMA and flush TX queues */
	dma_ctrl = bcmgenet_dma_disable(priv);

	/* Reinitialize TDMA and RDMA and SW housekeeping */
	ret = bcmgenet_init_dma(priv);
	if (ret) {
		netdev_err(dev, "failed to initialize DMA\n");
		goto err_clk_disable;
	}

	/* Always enable ring 16 - descriptor ring */
	bcmgenet_enable_dma(priv, dma_ctrl);

	/* HFB init */
	bcmgenet_hfb_init(priv);

	ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
			  dev->name, priv);
	if (ret < 0) {
		netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
		goto err_fini_dma;
	}

	ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
			  dev->name, priv);
	if (ret < 0) {
		netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
		goto err_irq0;
	}

	ret = bcmgenet_mii_probe(dev);
	if (ret) {
		netdev_err(dev, "failed to connect to PHY\n");
		goto err_irq1;
	}

	bcmgenet_netif_start(dev);

	netif_tx_start_all_queues(dev);

	return 0;

err_irq1:
	free_irq(priv->irq1, priv);
err_irq0:
	free_irq(priv->irq0, priv);
err_fini_dma:
	bcmgenet_dma_teardown(priv);
	bcmgenet_fini_dma(priv);
err_clk_disable:
	if (priv->internal_phy)
		bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
	clk_disable_unprepare(priv->clk);
	return ret;
}

static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);

	bcmgenet_disable_tx_napi(priv);
	netif_tx_disable(dev);

	/* Disable MAC receive */
	umac_enable_set(priv, CMD_RX_EN, false);

	bcmgenet_dma_teardown(priv);

	/* Disable MAC transmit. TX DMA disabled must be done before this */
	umac_enable_set(priv, CMD_TX_EN, false);

	if (stop_phy)
		phy_stop(dev->phydev);
	bcmgenet_disable_rx_napi(priv);
	bcmgenet_intr_disable(priv);

	/* Wait for pending work items to complete. Since interrupts are
	 * disabled no new work will be scheduled.
	 */
	cancel_work_sync(&priv->bcmgenet_irq_work);

	priv->old_link = -1;
	priv->old_speed = -1;
	priv->old_duplex = -1;
	priv->old_pause = -1;

	/* tx reclaim */
	bcmgenet_tx_reclaim_all(dev);
	bcmgenet_fini_dma(priv);
}

static int bcmgenet_close(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	int ret = 0;

	netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");

	bcmgenet_netif_stop(dev, false);

	/* Really kill the PHY state machine and disconnect from it */
	phy_disconnect(dev->phydev);

	free_irq(priv->irq0, priv);
	free_irq(priv->irq1, priv);

	if (priv->internal_phy)
		ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);

	clk_disable_unprepare(priv->clk);

	return ret;
}

static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)
{
	struct bcmgenet_priv *priv = ring->priv;
	u32 p_index, c_index, intsts, intmsk;
	struct netdev_queue *txq;
	unsigned int free_bds;
	bool txq_stopped;

	if (!netif_msg_tx_err(priv))
		return;

	txq = netdev_get_tx_queue(priv->dev, ring->queue);

	spin_lock(&ring->lock);
	if (ring->index == DESC_INDEX) {
		intsts = ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
		intmsk = UMAC_IRQ_TXDMA_DONE | UMAC_IRQ_TXDMA_MBDONE;
	} else {
		intsts = ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
		intmsk = 1 << ring->index;
	}
	c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
	p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);
	txq_stopped = netif_tx_queue_stopped(txq);
	free_bds = ring->free_bds;
	spin_unlock(&ring->lock);

	netif_err(priv, tx_err, priv->dev, "Ring %d queue %d status summary\n"
		  "TX queue status: %s, interrupts: %s\n"
		  "(sw)free_bds: %d (sw)size: %d\n"
		  "(sw)p_index: %d (hw)p_index: %d\n"
		  "(sw)c_index: %d (hw)c_index: %d\n"
		  "(sw)clean_p: %d (sw)write_p: %d\n"
		  "(sw)cb_ptr: %d (sw)end_ptr: %d\n",
		  ring->index, ring->queue,
		  txq_stopped ? "stopped" : "active",
		  intsts & intmsk ? "enabled" : "disabled",
		  free_bds, ring->size,
		  ring->prod_index, p_index & DMA_P_INDEX_MASK,
		  ring->c_index, c_index & DMA_C_INDEX_MASK,
		  ring->clean_ptr, ring->write_ptr,
		  ring->cb_ptr, ring->end_ptr);
}

static void bcmgenet_timeout(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	u32 int0_enable = 0;
	u32 int1_enable = 0;
	unsigned int q;

	netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");

	for (q = 0; q < priv->hw_params->tx_queues; q++)
		bcmgenet_dump_tx_queue(&priv->tx_rings[q]);
	bcmgenet_dump_tx_queue(&priv->tx_rings[DESC_INDEX]);

	bcmgenet_tx_reclaim_all(dev);

	for (q = 0; q < priv->hw_params->tx_queues; q++)
		int1_enable |= (1 << q);

	int0_enable = UMAC_IRQ_TXDMA_DONE;

	/* Re-enable TX interrupts if disabled */
	bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
	bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR);

	netif_trans_update(dev);

	dev->stats.tx_errors++;

	netif_tx_wake_all_queues(dev);
}

#define MAX_MDF_FILTER	17

static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
					 unsigned char *addr,
					 int *i)
{
	bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
			     UMAC_MDF_ADDR + (*i * 4));
	bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
			     addr[4] << 8 | addr[5],
			     UMAC_MDF_ADDR + ((*i + 1) * 4));
	*i += 2;
}

static void bcmgenet_set_rx_mode(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	struct netdev_hw_addr *ha;
	int i, nfilter;
	u32 reg;

	netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);

	/* Number of filters needed */
	nfilter = netdev_uc_count(dev) + netdev_mc_count(dev) + 2;

	/*
	 * Turn on promicuous mode for three scenarios
	 * 1. IFF_PROMISC flag is set
	 * 2. IFF_ALLMULTI flag is set
	 * 3. The number of filters needed exceeds the number filters
	 *    supported by the hardware.
	*/
	reg = bcmgenet_umac_readl(priv, UMAC_CMD);
	if ((dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) ||
	    (nfilter > MAX_MDF_FILTER)) {
		reg |= CMD_PROMISC;
		bcmgenet_umac_writel(priv, reg, UMAC_CMD);
		bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
		return;
	} else {
		reg &= ~CMD_PROMISC;
		bcmgenet_umac_writel(priv, reg, UMAC_CMD);
	}

	/* update MDF filter */
	i = 0;
	/* Broadcast */
	bcmgenet_set_mdf_addr(priv, dev->broadcast, &i);
	/* my own address.*/
	bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i);

	/* Unicast */
	netdev_for_each_uc_addr(ha, dev)
		bcmgenet_set_mdf_addr(priv, ha->addr, &i);

	/* Multicast */
	netdev_for_each_mc_addr(ha, dev)
		bcmgenet_set_mdf_addr(priv, ha->addr, &i);

	/* Enable filters */
	reg = GENMASK(MAX_MDF_FILTER - 1, MAX_MDF_FILTER - nfilter);
	bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
}

/* Set the hardware MAC address. */
static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
{
	struct sockaddr *addr = p;

	/* Setting the MAC address at the hardware level is not possible
	 * without disabling the UniMAC RX/TX enable bits.
	 */
	if (netif_running(dev))
		return -EBUSY;

	ether_addr_copy(dev->dev_addr, addr->sa_data);

	return 0;
}

static struct net_device_stats *bcmgenet_get_stats(struct net_device *dev)
{
	struct bcmgenet_priv *priv = netdev_priv(dev);
	unsigned long tx_bytes = 0, tx_packets = 0;
	unsigned long rx_bytes = 0, rx_packets = 0;
	unsigned long rx_errors = 0, rx_dropped = 0;
	struct bcmgenet_tx_ring *tx_ring;
	struct bcmgenet_rx_ring *rx_ring;
	unsigned int q;

	for (q = 0; q < priv->hw_params->tx_queues; q++) {
		tx_ring = &priv->tx_rings[q];
		tx_bytes += tx_ring->bytes;
		tx_packets += tx_ring->packets;
	}
	tx_ring = &priv->tx_rings[DESC_INDEX];
	tx_bytes += tx_ring->bytes;
	tx_packets += tx_ring->packets;

	for (q = 0; q < priv->hw_params->rx_queues; q++) {
		rx_ring = &priv->rx_rings[q];

		rx_bytes += rx_ring->bytes;
		rx_packets += rx_ring->packets;
		rx_errors += rx_ring->errors;
		rx_dropped += rx_ring->dropped;
	}
	rx_ring = &priv->rx_rings[DESC_INDEX];
	rx_bytes += rx_ring->bytes;
	rx_packets += rx_ring->packets;
	rx_errors += rx_ring->errors;
	rx_dropped += rx_ring->dropped;

	dev->stats.tx_bytes = tx_bytes;
	dev->stats.tx_packets = tx_packets;
	dev->stats.rx_bytes = rx_bytes;
	dev->stats.rx_packets = rx_packets;
	dev->stats.rx_errors = rx_errors;
	dev->stats.rx_missed_errors = rx_errors;
	dev->stats.rx_dropped = rx_dropped;
	return &dev->stats;
}

static const struct net_device_ops bcmgenet_netdev_ops = {
	.ndo_open		= bcmgenet_open,
	.ndo_stop		= bcmgenet_close,
	.ndo_start_xmit		= bcmgenet_xmit,
	.ndo_tx_timeout		= bcmgenet_timeout,
	.ndo_set_rx_mode	= bcmgenet_set_rx_mode,
	.ndo_set_mac_address	= bcmgenet_set_mac_addr,
	.ndo_do_ioctl		= bcmgenet_ioctl,
	.ndo_set_features	= bcmgenet_set_features,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= bcmgenet_poll_controller,
#endif
	.ndo_get_stats		= bcmgenet_get_stats,
};

/* Array of GENET hardware parameters/characteristics */
static struct bcmgenet_hw_params bcmgenet_hw_params[] = {
	[GENET_V1] = {
		.tx_queues = 0,
		.tx_bds_per_q = 0,
		.rx_queues = 0,
		.rx_bds_per_q = 0,
		.bp_in_en_shift = 16,
		.bp_in_mask = 0xffff,
		.hfb_filter_cnt = 16,
		.qtag_mask = 0x1F,
		.hfb_offset = 0x1000,
		.rdma_offset = 0x2000,
		.tdma_offset = 0x3000,
		.words_per_bd = 2,
	},
	[GENET_V2] = {
		.tx_queues = 4,
		.tx_bds_per_q = 32,
		.rx_queues = 0,
		.rx_bds_per_q = 0,
		.bp_in_en_shift = 16,
		.bp_in_mask = 0xffff,
		.hfb_filter_cnt = 16,
		.qtag_mask = 0x1F,
		.tbuf_offset = 0x0600,
		.hfb_offset = 0x1000,
		.hfb_reg_offset = 0x2000,
		.rdma_offset = 0x3000,
		.tdma_offset = 0x4000,
		.words_per_bd = 2,
		.flags = GENET_HAS_EXT,
	},
	[GENET_V3] = {
		.tx_queues = 4,
		.tx_bds_per_q = 32,
		.rx_queues = 0,
		.rx_bds_per_q = 0,
		.bp_in_en_shift = 17,
		.bp_in_mask = 0x1ffff,
		.hfb_filter_cnt = 48,
		.hfb_filter_size = 128,
		.qtag_mask = 0x3F,
		.tbuf_offset = 0x0600,
		.hfb_offset = 0x8000,
		.hfb_reg_offset = 0xfc00,
		.rdma_offset = 0x10000,
		.tdma_offset = 0x11000,
		.words_per_bd = 2,
		.flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR |
			 GENET_HAS_MOCA_LINK_DET,
	},
	[GENET_V4] = {
		.tx_queues = 4,
		.tx_bds_per_q = 32,
		.rx_queues = 0,
		.rx_bds_per_q = 0,
		.bp_in_en_shift = 17,
		.bp_in_mask = 0x1ffff,
		.hfb_filter_cnt = 48,
		.hfb_filter_size = 128,
		.qtag_mask = 0x3F,
		.tbuf_offset = 0x0600,
		.hfb_offset = 0x8000,
		.hfb_reg_offset = 0xfc00,
		.rdma_offset = 0x2000,
		.tdma_offset = 0x4000,
		.words_per_bd = 3,
		.flags = GENET_HAS_40BITS | GENET_HAS_EXT |
			 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
	},
	[GENET_V5] = {
		.tx_queues = 4,
		.tx_bds_per_q = 32,
		.rx_queues = 0,
		.rx_bds_per_q = 0,
		.bp_in_en_shift = 17,
		.bp_in_mask = 0x1ffff,
		.hfb_filter_cnt = 48,
		.hfb_filter_size = 128,
		.qtag_mask = 0x3F,
		.tbuf_offset = 0x0600,
		.hfb_offset = 0x8000,
		.hfb_reg_offset = 0xfc00,
		.rdma_offset = 0x2000,
		.tdma_offset = 0x4000,
		.words_per_bd = 3,
		.flags = GENET_HAS_40BITS | GENET_HAS_EXT |
			 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
	},
};

/* Infer hardware parameters from the detected GENET version */
static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
{
	struct bcmgenet_hw_params *params;
	u32 reg;
	u8 major;
	u16 gphy_rev;

	if (GENET_IS_V5(priv) || GENET_IS_V4(priv)) {
		bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
		genet_dma_ring_regs = genet_dma_ring_regs_v4;
		priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
	} else if (GENET_IS_V3(priv)) {
		bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
		genet_dma_ring_regs = genet_dma_ring_regs_v123;
		priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
	} else if (GENET_IS_V2(priv)) {
		bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
		genet_dma_ring_regs = genet_dma_ring_regs_v123;
		priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
	} else if (GENET_IS_V1(priv)) {
		bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
		genet_dma_ring_regs = genet_dma_ring_regs_v123;
		priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
	}

	/* enum genet_version starts at 1 */
	priv->hw_params = &bcmgenet_hw_params[priv->version];
	params = priv->hw_params;

	/* Read GENET HW version */
	reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
	major = (reg >> 24 & 0x0f);
	if (major == 6)
		major = 5;
	else if (major == 5)
		major = 4;
	else if (major == 0)
		major = 1;
	if (major != priv->version) {
		dev_err(&priv->pdev->dev,
			"GENET version mismatch, got: %d, configured for: %d\n",
			major, priv->version);
	}

	/* Print the GENET core version */
	dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
		 major, (reg >> 16) & 0x0f, reg & 0xffff);

	/* Store the integrated PHY revision for the MDIO probing function
	 * to pass this information to the PHY driver. The PHY driver expects
	 * to find the PHY major revision in bits 15:8 while the GENET register
	 * stores that information in bits 7:0, account for that.
	 *
	 * On newer chips, starting with PHY revision G0, a new scheme is
	 * deployed similar to the Starfighter 2 switch with GPHY major
	 * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
	 * is reserved as well as special value 0x01ff, we have a small
	 * heuristic to check for the new GPHY revision and re-arrange things
	 * so the GPHY driver is happy.
	 */
	gphy_rev = reg & 0xffff;

	if (GENET_IS_V5(priv)) {
		/* The EPHY revision should come from the MDIO registers of
		 * the PHY not from GENET.
		 */
		if (gphy_rev != 0) {
			pr_warn("GENET is reporting EPHY revision: 0x%04x\n",
				gphy_rev);
		}
	/* This is reserved so should require special treatment */
	} else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
		pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
		return;
	/* This is the good old scheme, just GPHY major, no minor nor patch */
	} else if ((gphy_rev & 0xf0) != 0) {
		priv->gphy_rev = gphy_rev << 8;
	/* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
	} else if ((gphy_rev & 0xff00) != 0) {
		priv->gphy_rev = gphy_rev;
	}

#ifdef CONFIG_PHYS_ADDR_T_64BIT
	if (!(params->flags & GENET_HAS_40BITS))
		pr_warn("GENET does not support 40-bits PA\n");
#endif

	pr_debug("Configuration for version: %d\n"
		"TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
		"BP << en: %2d, BP msk: 0x%05x\n"
		"HFB count: %2d, QTAQ msk: 0x%05x\n"
		"TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
		"RDMA: 0x%05x, TDMA: 0x%05x\n"
		"Words/BD: %d\n",
		priv->version,
		params->tx_queues, params->tx_bds_per_q,
		params->rx_queues, params->rx_bds_per_q,
		params->bp_in_en_shift, params->bp_in_mask,
		params->hfb_filter_cnt, params->qtag_mask,
		params->tbuf_offset, params->hfb_offset,
		params->hfb_reg_offset,
		params->rdma_offset, params->tdma_offset,
		params->words_per_bd);
}

static const struct of_device_id bcmgenet_match[] = {
	{ .compatible = "brcm,genet-v1", .data = (void *)GENET_V1 },
	{ .compatible = "brcm,genet-v2", .data = (void *)GENET_V2 },
	{ .compatible = "brcm,genet-v3", .data = (void *)GENET_V3 },
	{ .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 },
	{ .compatible = "brcm,genet-v5", .data = (void *)GENET_V5 },
	{ },
};
MODULE_DEVICE_TABLE(of, bcmgenet_match);

static int bcmgenet_probe(struct platform_device *pdev)
{
	struct bcmgenet_platform_data *pd = pdev->dev.platform_data;
	struct device_node *dn = pdev->dev.of_node;
	const struct of_device_id *of_id = NULL;
	struct bcmgenet_priv *priv;
	struct net_device *dev;
	const void *macaddr;
	unsigned int i;
	int err = -EIO;
	const char *phy_mode_str;

	/* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
	dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
				 GENET_MAX_MQ_CNT + 1);
	if (!dev) {
		dev_err(&pdev->dev, "can't allocate net device\n");
		return -ENOMEM;
	}

	if (dn) {
		of_id = of_match_node(bcmgenet_match, dn);
		if (!of_id)
			return -EINVAL;
	}

	priv = netdev_priv(dev);
	priv->irq0 = platform_get_irq(pdev, 0);
	priv->irq1 = platform_get_irq(pdev, 1);
	priv->wol_irq = platform_get_irq(pdev, 2);
	if (!priv->irq0 || !priv->irq1) {
		dev_err(&pdev->dev, "can't find IRQs\n");
		err = -EINVAL;
		goto err;
	}

	if (dn) {
		macaddr = of_get_mac_address(dn);
		if (IS_ERR(macaddr)) {
			dev_err(&pdev->dev, "can't find MAC address\n");
			err = -EINVAL;
			goto err;
		}
	} else {
		macaddr = pd->mac_address;
	}

	priv->base = devm_platform_ioremap_resource(pdev, 0);
	if (IS_ERR(priv->base)) {
		err = PTR_ERR(priv->base);
		goto err;
	}

	spin_lock_init(&priv->lock);

	SET_NETDEV_DEV(dev, &pdev->dev);
	dev_set_drvdata(&pdev->dev, dev);
	ether_addr_copy(dev->dev_addr, macaddr);
	dev->watchdog_timeo = 2 * HZ;
	dev->ethtool_ops = &bcmgenet_ethtool_ops;
	dev->netdev_ops = &bcmgenet_netdev_ops;

	priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);

	/* Set hardware features */
	dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM |
		NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;

	/* Request the WOL interrupt and advertise suspend if available */
	priv->wol_irq_disabled = true;
	if (priv->wol_irq > 0) {
		err = devm_request_irq(&pdev->dev, priv->wol_irq,
				       bcmgenet_wol_isr, 0, dev->name, priv);
		if (!err)
			device_set_wakeup_capable(&pdev->dev, 1);
	}

	/* Set the needed headroom to account for any possible
	 * features enabling/disabling at runtime
	 */
	dev->needed_headroom += 64;

	netdev_boot_setup_check(dev);

	priv->dev = dev;
	priv->pdev = pdev;
	if (of_id)
		priv->version = (enum bcmgenet_version)of_id->data;
	else
		priv->version = pd->genet_version;

	priv->clk = devm_clk_get(&priv->pdev->dev, "enet");
	if (IS_ERR(priv->clk)) {
		dev_warn(&priv->pdev->dev, "failed to get enet clock\n");
		priv->clk = NULL;
	}

	clk_prepare_enable(priv->clk);

	bcmgenet_set_hw_params(priv);

	/* Mii wait queue */
	init_waitqueue_head(&priv->wq);
	/* Always use RX_BUF_LENGTH (2KB) buffer for all chips */
	priv->rx_buf_len = RX_BUF_LENGTH;
	INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);

	priv->clk_wol = devm_clk_get(&priv->pdev->dev, "enet-wol");
	if (IS_ERR(priv->clk_wol)) {
		dev_warn(&priv->pdev->dev, "failed to get enet-wol clock\n");
		priv->clk_wol = NULL;
	}

	priv->clk_eee = devm_clk_get(&priv->pdev->dev, "enet-eee");
	if (IS_ERR(priv->clk_eee)) {
		dev_warn(&priv->pdev->dev, "failed to get enet-eee clock\n");
		priv->clk_eee = NULL;
	}

	/* If this is an internal GPHY, power it on now, before UniMAC is
	 * brought out of reset as absolutely no UniMAC activity is allowed
	 */
	if (dn && !of_property_read_string(dn, "phy-mode", &phy_mode_str) &&
	    !strcasecmp(phy_mode_str, "internal"))
		bcmgenet_power_up(priv, GENET_POWER_PASSIVE);

	reset_umac(priv);

	err = bcmgenet_mii_init(dev);
	if (err)
		goto err_clk_disable;

	/* setup number of real queues  + 1 (GENET_V1 has 0 hardware queues
	 * just the ring 16 descriptor based TX
	 */
	netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
	netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);

	/* Set default coalescing parameters */
	for (i = 0; i < priv->hw_params->rx_queues; i++)
		priv->rx_rings[i].rx_max_coalesced_frames = 1;
	priv->rx_rings[DESC_INDEX].rx_max_coalesced_frames = 1;

	/* libphy will determine the link state */
	netif_carrier_off(dev);

	/* Turn off the main clock, WOL clock is handled separately */
	clk_disable_unprepare(priv->clk);

	err = register_netdev(dev);
	if (err) {
		bcmgenet_mii_exit(dev);
		goto err;
	}

	return err;

err_clk_disable:
	clk_disable_unprepare(priv->clk);
err:
	free_netdev(dev);
	return err;
}

static int bcmgenet_remove(struct platform_device *pdev)
{
	struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);

	dev_set_drvdata(&pdev->dev, NULL);
	unregister_netdev(priv->dev);
	bcmgenet_mii_exit(priv->dev);
	free_netdev(priv->dev);

	return 0;
}

#ifdef CONFIG_PM_SLEEP
static int bcmgenet_resume(struct device *d)
{
	struct net_device *dev = dev_get_drvdata(d);
	struct bcmgenet_priv *priv = netdev_priv(dev);
	unsigned long dma_ctrl;
	int ret;

	if (!netif_running(dev))
		return 0;

	/* Turn on the clock */
	ret = clk_prepare_enable(priv->clk);
	if (ret)
		return ret;

	/* If this is an internal GPHY, power it back on now, before UniMAC is
	 * brought out of reset as absolutely no UniMAC activity is allowed
	 */
	if (priv->internal_phy)
		bcmgenet_power_up(priv, GENET_POWER_PASSIVE);

	bcmgenet_umac_reset(priv);

	init_umac(priv);

	/* From WOL-enabled suspend, switch to regular clock */
	if (priv->wolopts)
		clk_disable_unprepare(priv->clk_wol);

	phy_init_hw(dev->phydev);

	/* Speed settings must be restored */
	genphy_config_aneg(dev->phydev);
	bcmgenet_mii_config(priv->dev, false);

	bcmgenet_set_hw_addr(priv, dev->dev_addr);

	if (priv->wolopts)
		bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);

	/* Disable RX/TX DMA and flush TX queues */
	dma_ctrl = bcmgenet_dma_disable(priv);

	/* Reinitialize TDMA and RDMA and SW housekeeping */
	ret = bcmgenet_init_dma(priv);
	if (ret) {
		netdev_err(dev, "failed to initialize DMA\n");
		goto out_clk_disable;
	}

	/* Always enable ring 16 - descriptor ring */
	bcmgenet_enable_dma(priv, dma_ctrl);

	if (!device_may_wakeup(d))
		phy_resume(dev->phydev);

	bcmgenet_netif_start(dev);

	netif_device_attach(dev);

	return 0;

out_clk_disable:
	if (priv->internal_phy)
		bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
	clk_disable_unprepare(priv->clk);
	return ret;
}

static int bcmgenet_suspend(struct device *d)
{
	struct net_device *dev = dev_get_drvdata(d);
	struct bcmgenet_priv *priv = netdev_priv(dev);
	int ret = 0;

	if (!netif_running(dev))
		return 0;

	netif_device_detach(dev);

	bcmgenet_netif_stop(dev, true);

	if (!device_may_wakeup(d))
		phy_suspend(dev->phydev);

	/* Prepare the device for Wake-on-LAN and switch to the slow clock */
	if (device_may_wakeup(d) && priv->wolopts) {
		ret = bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
		clk_prepare_enable(priv->clk_wol);
	} else if (priv->internal_phy) {
		ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
	}

	/* Turn off the clocks */
	clk_disable_unprepare(priv->clk);

	if (ret)
		bcmgenet_resume(d);

	return ret;
}
#endif /* CONFIG_PM_SLEEP */

static SIMPLE_DEV_PM_OPS(bcmgenet_pm_ops, bcmgenet_suspend, bcmgenet_resume);

static struct platform_driver bcmgenet_driver = {
	.probe	= bcmgenet_probe,
	.remove	= bcmgenet_remove,
	.driver	= {
		.name	= "bcmgenet",
		.of_match_table = bcmgenet_match,
		.pm	= &bcmgenet_pm_ops,
	},
};
module_platform_driver(bcmgenet_driver);

MODULE_AUTHOR("Broadcom Corporation");
MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
MODULE_ALIAS("platform:bcmgenet");
MODULE_LICENSE("GPL");
MODULE_SOFTDEP("pre: mdio-bcm-unimac");