Linux Audio

Check our new training course

Embedded Linux Audio

Check our new training course
with Creative Commons CC-BY-SA
lecture materials

Bootlin logo

Elixir Cross Referencer

Loading...
   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
/*
 *  arch/s390/kernel/entry.S
 *    S390 low-level entry points.
 *
 *    Copyright (C) IBM Corp. 1999,2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
 *		 Hartmut Penner (hp@de.ibm.com),
 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
 */

#include <linux/sys.h>
#include <linux/linkage.h>
#include <linux/init.h>
#include <asm/cache.h>
#include <asm/lowcore.h>
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
#include <asm/page.h>

/*
 * Stack layout for the system_call stack entry.
 * The first few entries are identical to the user_regs_struct.
 */
SP_PTREGS    =	STACK_FRAME_OVERHEAD
SP_ARGS      =	STACK_FRAME_OVERHEAD + __PT_ARGS
SP_PSW	     =	STACK_FRAME_OVERHEAD + __PT_PSW
SP_R0	     =	STACK_FRAME_OVERHEAD + __PT_GPRS
SP_R1	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 4
SP_R2	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 8
SP_R3	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 12
SP_R4	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 16
SP_R5	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 20
SP_R6	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 24
SP_R7	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 28
SP_R8	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 32
SP_R9	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 36
SP_R10	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 40
SP_R11	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 44
SP_R12	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 48
SP_R13	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 52
SP_R14	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 56
SP_R15	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 60
SP_ORIG_R2   =	STACK_FRAME_OVERHEAD + __PT_ORIG_GPR2
SP_ILC	     =	STACK_FRAME_OVERHEAD + __PT_ILC
SP_TRAP      =	STACK_FRAME_OVERHEAD + __PT_TRAP
SP_SIZE      =	STACK_FRAME_OVERHEAD + __PT_SIZE

_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
		 _TIF_MCCK_PENDING | _TIF_RESTART_SVC | _TIF_SINGLE_STEP )
_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
		 _TIF_MCCK_PENDING)

STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
STACK_SIZE  = 1 << STACK_SHIFT

#define BASED(name) name-system_call(%r13)

#ifdef CONFIG_TRACE_IRQFLAGS
	.macro	TRACE_IRQS_ON
	l	%r1,BASED(.Ltrace_irq_on)
	basr	%r14,%r1
	.endm

	.macro	TRACE_IRQS_OFF
	l	%r1,BASED(.Ltrace_irq_off)
	basr	%r14,%r1
	.endm

	.macro	TRACE_IRQS_CHECK
	tm	SP_PSW(%r15),0x03	# irqs enabled?
	jz	0f
	l	%r1,BASED(.Ltrace_irq_on)
	basr	%r14,%r1
	j	1f
0:	l	%r1,BASED(.Ltrace_irq_off)
	basr	%r14,%r1
1:
	.endm
#else
#define TRACE_IRQS_ON
#define TRACE_IRQS_OFF
#define TRACE_IRQS_CHECK
#endif

#ifdef CONFIG_LOCKDEP
	.macro	LOCKDEP_SYS_EXIT
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
	jz	0f
	l	%r1,BASED(.Llockdep_sys_exit)
	basr	%r14,%r1
0:
	.endm
#else
#define LOCKDEP_SYS_EXIT
#endif

/*
 * Register usage in interrupt handlers:
 *    R9  - pointer to current task structure
 *    R13 - pointer to literal pool
 *    R14 - return register for function calls
 *    R15 - kernel stack pointer
 */

	.macro	STORE_TIMER lc_offset
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	stpt	\lc_offset
#endif
	.endm

#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.macro	UPDATE_VTIME lc_from,lc_to,lc_sum
	lm	%r10,%r11,\lc_from
	sl	%r10,\lc_to
	sl	%r11,\lc_to+4
	bc	3,BASED(0f)
	sl	%r10,BASED(.Lc_1)
0:	al	%r10,\lc_sum
	al	%r11,\lc_sum+4
	bc	12,BASED(1f)
	al	%r10,BASED(.Lc_1)
1:	stm	%r10,%r11,\lc_sum
	.endm
#endif

	.macro	SAVE_ALL_BASE savearea
	stm	%r12,%r15,\savearea
	l	%r13,__LC_SVC_NEW_PSW+4	# load &system_call to %r13
	.endm

	.macro	SAVE_ALL_SVC psworg,savearea
	la	%r12,\psworg
	l	%r15,__LC_KERNEL_STACK	# problem state -> load ksp
	.endm

	.macro	SAVE_ALL_SYNC psworg,savearea
	la	%r12,\psworg
	tm	\psworg+1,0x01		# test problem state bit
	bz	BASED(2f)		# skip stack setup save
	l	%r15,__LC_KERNEL_STACK	# problem state -> load ksp
#ifdef CONFIG_CHECK_STACK
	b	BASED(3f)
2:	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
	bz	BASED(stack_overflow)
3:
#endif
2:
	.endm

	.macro	SAVE_ALL_ASYNC psworg,savearea
	la	%r12,\psworg
	tm	\psworg+1,0x01		# test problem state bit
	bnz	BASED(1f)		# from user -> load async stack
	clc	\psworg+4(4),BASED(.Lcritical_end)
	bhe	BASED(0f)
	clc	\psworg+4(4),BASED(.Lcritical_start)
	bl	BASED(0f)
	l	%r14,BASED(.Lcleanup_critical)
	basr	%r14,%r14
	tm	1(%r12),0x01		# retest problem state after cleanup
	bnz	BASED(1f)
0:	l	%r14,__LC_ASYNC_STACK	# are we already on the async stack ?
	slr	%r14,%r15
	sra	%r14,STACK_SHIFT
	be	BASED(2f)
1:	l	%r15,__LC_ASYNC_STACK
#ifdef CONFIG_CHECK_STACK
	b	BASED(3f)
2:	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
	bz	BASED(stack_overflow)
3:
#endif
2:
	.endm

	.macro	CREATE_STACK_FRAME psworg,savearea
	s	%r15,BASED(.Lc_spsize)	# make room for registers & psw
	mvc	SP_PSW(8,%r15),0(%r12)	# move user PSW to stack
	la	%r12,\psworg
	st	%r2,SP_ORIG_R2(%r15)	# store original content of gpr 2
	icm	%r12,12,__LC_SVC_ILC
	stm	%r0,%r11,SP_R0(%r15)	# store gprs %r0-%r11 to kernel stack
	st	%r12,SP_ILC(%r15)
	mvc	SP_R12(16,%r15),\savearea # move %r12-%r15 to stack
	la	%r12,0
	st	%r12,__SF_BACKCHAIN(%r15)	# clear back chain
	.endm

	.macro	RESTORE_ALL psworg,sync
	mvc	\psworg(8),SP_PSW(%r15) # move user PSW to lowcore
	.if !\sync
	ni	\psworg+1,0xfd		# clear wait state bit
	.endif
	lm	%r0,%r15,SP_R0(%r15)	# load gprs 0-15 of user
	STORE_TIMER __LC_EXIT_TIMER
	lpsw	\psworg			# back to caller
	.endm

/*
 * Scheduler resume function, called by switch_to
 *  gpr2 = (task_struct *) prev
 *  gpr3 = (task_struct *) next
 * Returns:
 *  gpr2 = prev
 */
	.globl	__switch_to
__switch_to:
	basr	%r1,0
__switch_to_base:
	tm	__THREAD_per(%r3),0xe8		# new process is using per ?
	bz	__switch_to_noper-__switch_to_base(%r1)	# if not we're fine
	stctl	%c9,%c11,__SF_EMPTY(%r15)	# We are using per stuff
	clc	__THREAD_per(12,%r3),__SF_EMPTY(%r15)
	be	__switch_to_noper-__switch_to_base(%r1)	# we got away w/o bashing TLB's
	lctl	%c9,%c11,__THREAD_per(%r3)	# Nope we didn't
__switch_to_noper:
	l	%r4,__THREAD_info(%r2)		# get thread_info of prev
	tm	__TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
	bz	__switch_to_no_mcck-__switch_to_base(%r1)
	ni	__TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
	l	%r4,__THREAD_info(%r3)		# get thread_info of next
	oi	__TI_flags+3(%r4),_TIF_MCCK_PENDING # set it in next
__switch_to_no_mcck:
	stm	%r6,%r15,__SF_GPRS(%r15)# store __switch_to registers of prev task
	st	%r15,__THREAD_ksp(%r2)	# store kernel stack to prev->tss.ksp
	l	%r15,__THREAD_ksp(%r3)	# load kernel stack from next->tss.ksp
	lm	%r6,%r15,__SF_GPRS(%r15)# load __switch_to registers of next task
	st	%r3,__LC_CURRENT	# __LC_CURRENT = current task struct
	lctl	%c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
	l	%r3,__THREAD_info(%r3)	# load thread_info from task struct
	st	%r3,__LC_THREAD_INFO
	ahi	%r3,STACK_SIZE
	st	%r3,__LC_KERNEL_STACK	# __LC_KERNEL_STACK = new kernel stack
	br	%r14

__critical_start:
/*
 * SVC interrupt handler routine. System calls are synchronous events and
 * are executed with interrupts enabled.
 */

	.globl	system_call
system_call:
	STORE_TIMER __LC_SYNC_ENTER_TIMER
sysc_saveall:
	SAVE_ALL_BASE __LC_SAVE_AREA
	SAVE_ALL_SVC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	lh	%r7,0x8a	  # get svc number from lowcore
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
sysc_vtime:
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
sysc_stime:
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
sysc_update:
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
#endif
sysc_do_svc:
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	sla	%r7,2			# *4 and test for svc 0
	bnz	BASED(sysc_nr_ok)	# svc number > 0
	# svc 0: system call number in %r1
	cl	%r1,BASED(.Lnr_syscalls)
	bnl	BASED(sysc_nr_ok)
	lr	%r7,%r1 	  # copy svc number to %r7
	sla	%r7,2		  # *4
sysc_nr_ok:
	mvc	SP_ARGS(4,%r15),SP_R7(%r15)
sysc_do_restart:
	l	%r8,BASED(.Lsysc_table)
	tm	__TI_flags+3(%r9),(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)
	l	%r8,0(%r7,%r8)	  # get system call addr.
	bnz	BASED(sysc_tracesys)
	basr	%r14,%r8	  # call sys_xxxx
	st	%r2,SP_R2(%r15)   # store return value (change R2 on stack)

sysc_return:
	tm	__TI_flags+3(%r9),_TIF_WORK_SVC
	bnz	BASED(sysc_work)  # there is work to do (signals etc.)
sysc_restore:
#ifdef CONFIG_TRACE_IRQFLAGS
	la	%r1,BASED(sysc_restore_trace_psw)
	lpsw	0(%r1)
sysc_restore_trace:
	TRACE_IRQS_CHECK
	LOCKDEP_SYS_EXIT
#endif
sysc_leave:
	RESTORE_ALL __LC_RETURN_PSW,1
sysc_done:

#ifdef CONFIG_TRACE_IRQFLAGS
	.align	8
	.globl	sysc_restore_trace_psw
sysc_restore_trace_psw:
	.long	0, sysc_restore_trace + 0x80000000
#endif

#
# recheck if there is more work to do
#
sysc_work_loop:
	tm	__TI_flags+3(%r9),_TIF_WORK_SVC
	bz	BASED(sysc_restore)	# there is no work to do
#
# One of the work bits is on. Find out which one.
#
sysc_work:
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
	bno	BASED(sysc_restore)
	tm	__TI_flags+3(%r9),_TIF_MCCK_PENDING
	bo	BASED(sysc_mcck_pending)
	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
	bo	BASED(sysc_reschedule)
	tm	__TI_flags+3(%r9),_TIF_SIGPENDING
	bnz	BASED(sysc_sigpending)
	tm	__TI_flags+3(%r9),_TIF_RESTART_SVC
	bo	BASED(sysc_restart)
	tm	__TI_flags+3(%r9),_TIF_SINGLE_STEP
	bo	BASED(sysc_singlestep)
	b	BASED(sysc_restore)
sysc_work_done:

#
# _TIF_NEED_RESCHED is set, call schedule
#
sysc_reschedule:
	l	%r1,BASED(.Lschedule)
	la	%r14,BASED(sysc_work_loop)
	br	%r1			# call scheduler

#
# _TIF_MCCK_PENDING is set, call handler
#
sysc_mcck_pending:
	l	%r1,BASED(.Ls390_handle_mcck)
	la	%r14,BASED(sysc_work_loop)
	br	%r1			# TIF bit will be cleared by handler

#
# _TIF_SIGPENDING is set, call do_signal
#
sysc_sigpending:
	ni	__TI_flags+3(%r9),255-_TIF_SINGLE_STEP # clear TIF_SINGLE_STEP
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	l	%r1,BASED(.Ldo_signal)
	basr	%r14,%r1		# call do_signal
	tm	__TI_flags+3(%r9),_TIF_RESTART_SVC
	bo	BASED(sysc_restart)
	tm	__TI_flags+3(%r9),_TIF_SINGLE_STEP
	bo	BASED(sysc_singlestep)
	b	BASED(sysc_work_loop)

#
# _TIF_RESTART_SVC is set, set up registers and restart svc
#
sysc_restart:
	ni	__TI_flags+3(%r9),255-_TIF_RESTART_SVC # clear TIF_RESTART_SVC
	l	%r7,SP_R2(%r15) 	# load new svc number
	sla	%r7,2
	mvc	SP_R2(4,%r15),SP_ORIG_R2(%r15) # restore first argument
	lm	%r2,%r6,SP_R2(%r15)	# load svc arguments
	b	BASED(sysc_do_restart)	# restart svc

#
# _TIF_SINGLE_STEP is set, call do_single_step
#
sysc_singlestep:
	ni	__TI_flags+3(%r9),255-_TIF_SINGLE_STEP # clear TIF_SINGLE_STEP
	mvi	SP_TRAP+1(%r15),0x28	# set trap indication to pgm check
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	l	%r1,BASED(.Lhandle_per)	# load adr. of per handler
	la	%r14,BASED(sysc_return)	# load adr. of system return
	br	%r1			# branch to do_single_step

#
# call trace before and after sys_call
#
sysc_tracesys:
	l	%r1,BASED(.Ltrace)
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	la	%r3,0
	srl	%r7,2
	st	%r7,SP_R2(%r15)
	basr	%r14,%r1
	clc	SP_R2(4,%r15),BASED(.Lnr_syscalls)
	bnl	BASED(sysc_tracenogo)
	l	%r8,BASED(.Lsysc_table)
	l	%r7,SP_R2(%r15) 	# strace might have changed the
	sll	%r7,2			#  system call
	l	%r8,0(%r7,%r8)
sysc_tracego:
	lm	%r3,%r6,SP_R3(%r15)
	l	%r2,SP_ORIG_R2(%r15)
	basr	%r14,%r8		# call sys_xxx
	st	%r2,SP_R2(%r15)		# store return value
sysc_tracenogo:
	tm	__TI_flags+3(%r9),(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)
	bz	BASED(sysc_return)
	l	%r1,BASED(.Ltrace)
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	la	%r3,1
	la	%r14,BASED(sysc_return)
	br	%r1

#
# a new process exits the kernel with ret_from_fork
#
	.globl	ret_from_fork
ret_from_fork:
	l	%r13,__LC_SVC_NEW_PSW+4
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	tm	SP_PSW+1(%r15),0x01	# forking a kernel thread ?
	bo	BASED(0f)
	st	%r15,SP_R15(%r15)	# store stack pointer for new kthread
0:	l	%r1,BASED(.Lschedtail)
	basr	%r14,%r1
	TRACE_IRQS_ON
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	b	BASED(sysc_return)

#
# kernel_execve function needs to deal with pt_regs that is not
# at the usual place
#
	.globl	kernel_execve
kernel_execve:
	stm	%r12,%r15,48(%r15)
	lr	%r14,%r15
	l	%r13,__LC_SVC_NEW_PSW+4
	s	%r15,BASED(.Lc_spsize)
	st	%r14,__SF_BACKCHAIN(%r15)
	la	%r12,SP_PTREGS(%r15)
	xc	0(__PT_SIZE,%r12),0(%r12)
	l	%r1,BASED(.Ldo_execve)
	lr	%r5,%r12
	basr	%r14,%r1
	ltr	%r2,%r2
	be	BASED(0f)
	a	%r15,BASED(.Lc_spsize)
	lm	%r12,%r15,48(%r15)
	br	%r14
	# execve succeeded.
0:	stnsm	__SF_EMPTY(%r15),0xfc	# disable interrupts
	l	%r15,__LC_KERNEL_STACK	# load ksp
	s	%r15,BASED(.Lc_spsize)	# make room for registers & psw
	l	%r9,__LC_THREAD_INFO
	mvc	SP_PTREGS(__PT_SIZE,%r15),0(%r12)	# copy pt_regs
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	l	%r1,BASED(.Lexecve_tail)
	basr	%r14,%r1
	b	BASED(sysc_return)

/*
 * Program check handler routine
 */

	.globl	pgm_check_handler
pgm_check_handler:
/*
 * First we need to check for a special case:
 * Single stepping an instruction that disables the PER event mask will
 * cause a PER event AFTER the mask has been set. Example: SVC or LPSW.
 * For a single stepped SVC the program check handler gets control after
 * the SVC new PSW has been loaded. But we want to execute the SVC first and
 * then handle the PER event. Therefore we update the SVC old PSW to point
 * to the pgm_check_handler and branch to the SVC handler after we checked
 * if we have to load the kernel stack register.
 * For every other possible cause for PER event without the PER mask set
 * we just ignore the PER event (FIXME: is there anything we have to do
 * for LPSW?).
 */
	STORE_TIMER __LC_SYNC_ENTER_TIMER
	SAVE_ALL_BASE __LC_SAVE_AREA
	tm	__LC_PGM_INT_CODE+1,0x80 # check whether we got a per exception
	bnz	BASED(pgm_per)		# got per exception -> special case
	SAVE_ALL_SYNC __LC_PGM_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_PGM_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	bz	BASED(pgm_no_vtime)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
pgm_no_vtime:
#endif
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	l	%r3,__LC_PGM_ILC	# load program interruption code
	la	%r8,0x7f
	nr	%r8,%r3
pgm_do_call:
	l	%r7,BASED(.Ljump_table)
	sll	%r8,2
	l	%r7,0(%r8,%r7)		# load address of handler routine
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	la	%r14,BASED(sysc_return)
	br	%r7			# branch to interrupt-handler

#
# handle per exception
#
pgm_per:
	tm	__LC_PGM_OLD_PSW,0x40	# test if per event recording is on
	bnz	BASED(pgm_per_std)	# ok, normal per event from user space
# ok its one of the special cases, now we need to find out which one
	clc	__LC_PGM_OLD_PSW(8),__LC_SVC_NEW_PSW
	be	BASED(pgm_svcper)
# no interesting special case, ignore PER event
	lm	%r12,%r15,__LC_SAVE_AREA
	lpsw	0x28

#
# Normal per exception
#
pgm_per_std:
	SAVE_ALL_SYNC __LC_PGM_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_PGM_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	bz	BASED(pgm_no_vtime2)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
pgm_no_vtime2:
#endif
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	l	%r1,__TI_task(%r9)
	mvc	__THREAD_per+__PER_atmid(2,%r1),__LC_PER_ATMID
	mvc	__THREAD_per+__PER_address(4,%r1),__LC_PER_ADDRESS
	mvc	__THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID
	oi	__TI_flags+3(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP
	tm	SP_PSW+1(%r15),0x01	# kernel per event ?
	bz	BASED(kernel_per)
	l	%r3,__LC_PGM_ILC	# load program interruption code
	la	%r8,0x7f
	nr	%r8,%r3 		# clear per-event-bit and ilc
	be	BASED(sysc_return)	# only per or per+check ?
	b	BASED(pgm_do_call)

#
# it was a single stepped SVC that is causing all the trouble
#
pgm_svcper:
	SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
#endif
	lh	%r7,0x8a		# get svc number from lowcore
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	l	%r1,__TI_task(%r9)
	mvc	__THREAD_per+__PER_atmid(2,%r1),__LC_PER_ATMID
	mvc	__THREAD_per+__PER_address(4,%r1),__LC_PER_ADDRESS
	mvc	__THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID
	oi	__TI_flags+3(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP
	TRACE_IRQS_ON
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	b	BASED(sysc_do_svc)

#
# per was called from kernel, must be kprobes
#
kernel_per:
	mvi	SP_TRAP+1(%r15),0x28	# set trap indication to pgm check
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	l	%r1,BASED(.Lhandle_per)	# load adr. of per handler
	la	%r14,BASED(sysc_restore)# load adr. of system return
	br	%r1			# branch to do_single_step

/*
 * IO interrupt handler routine
 */

	.globl io_int_handler
io_int_handler:
	STORE_TIMER __LC_ASYNC_ENTER_TIMER
	stck	__LC_INT_CLOCK
	SAVE_ALL_BASE __LC_SAVE_AREA+16
	SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+16
	CREATE_STACK_FRAME __LC_IO_OLD_PSW,__LC_SAVE_AREA+16
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	bz	BASED(io_no_vtime)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
io_no_vtime:
#endif
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	l	%r1,BASED(.Ldo_IRQ)	# load address of do_IRQ
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	basr	%r14,%r1		# branch to standard irq handler
io_return:
	tm	__TI_flags+3(%r9),_TIF_WORK_INT
	bnz	BASED(io_work)		# there is work to do (signals etc.)
io_restore:
#ifdef CONFIG_TRACE_IRQFLAGS
	la	%r1,BASED(io_restore_trace_psw)
	lpsw	0(%r1)
io_restore_trace:
	TRACE_IRQS_CHECK
	LOCKDEP_SYS_EXIT
#endif
io_leave:
	RESTORE_ALL __LC_RETURN_PSW,0
io_done:

#ifdef CONFIG_TRACE_IRQFLAGS
	.align	8
	.globl	io_restore_trace_psw
io_restore_trace_psw:
	.long	0, io_restore_trace + 0x80000000
#endif

#
# switch to kernel stack, then check the TIF bits
#
io_work:
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
#ifndef CONFIG_PREEMPT
	bno	BASED(io_restore)	# no-> skip resched & signal
#else
	bnz	BASED(io_work_user)	# no -> check for preemptive scheduling
	# check for preemptive scheduling
	icm	%r0,15,__TI_precount(%r9)
	bnz	BASED(io_restore)	# preemption disabled
	l	%r1,SP_R15(%r15)
	s	%r1,BASED(.Lc_spsize)
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lr	%r15,%r1
io_resume_loop:
	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
	bno	BASED(io_restore)
	l	%r1,BASED(.Lpreempt_schedule_irq)
	la	%r14,BASED(io_resume_loop)
	br	%r1			# call schedule
#endif

io_work_user:
	l	%r1,__LC_KERNEL_STACK
	s	%r1,BASED(.Lc_spsize)
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lr	%r15,%r1
#
# One of the work bits is on. Find out which one.
# Checked are: _TIF_SIGPENDING, _TIF_NEED_RESCHED
#		and _TIF_MCCK_PENDING
#
io_work_loop:
	tm	__TI_flags+3(%r9),_TIF_MCCK_PENDING
	bo	BASED(io_mcck_pending)
	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
	bo	BASED(io_reschedule)
	tm	__TI_flags+3(%r9),_TIF_SIGPENDING
	bnz	BASED(io_sigpending)
	b	BASED(io_restore)
io_work_done:

#
# _TIF_MCCK_PENDING is set, call handler
#
io_mcck_pending:
	l	%r1,BASED(.Ls390_handle_mcck)
	basr	%r14,%r1		# TIF bit will be cleared by handler
	b	BASED(io_work_loop)

#
# _TIF_NEED_RESCHED is set, call schedule
#
io_reschedule:
	TRACE_IRQS_ON
	l	%r1,BASED(.Lschedule)
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	basr	%r14,%r1		# call scheduler
	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
	TRACE_IRQS_OFF
	tm	__TI_flags+3(%r9),_TIF_WORK_INT
	bz	BASED(io_restore)	# there is no work to do
	b	BASED(io_work_loop)

#
# _TIF_SIGPENDING is set, call do_signal
#
io_sigpending:
	TRACE_IRQS_ON
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	l	%r1,BASED(.Ldo_signal)
	basr	%r14,%r1		# call do_signal
	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
	TRACE_IRQS_OFF
	b	BASED(io_work_loop)

/*
 * External interrupt handler routine
 */

	.globl	ext_int_handler
ext_int_handler:
	STORE_TIMER __LC_ASYNC_ENTER_TIMER
	stck	__LC_INT_CLOCK
	SAVE_ALL_BASE __LC_SAVE_AREA+16
	SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+16
	CREATE_STACK_FRAME __LC_EXT_OLD_PSW,__LC_SAVE_AREA+16
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	bz	BASED(ext_no_vtime)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
ext_no_vtime:
#endif
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	lh	%r3,__LC_EXT_INT_CODE	# get interruption code
	l	%r1,BASED(.Ldo_extint)
	basr	%r14,%r1
	b	BASED(io_return)

__critical_end:

/*
 * Machine check handler routines
 */

	.globl mcck_int_handler
mcck_int_handler:
	spt	__LC_CPU_TIMER_SAVE_AREA	# revalidate cpu timer
	lm	%r0,%r15,__LC_GPREGS_SAVE_AREA	# revalidate gprs
	SAVE_ALL_BASE __LC_SAVE_AREA+32
	la	%r12,__LC_MCK_OLD_PSW
	tm	__LC_MCCK_CODE,0x80	# system damage?
	bo	BASED(mcck_int_main)	# yes -> rest of mcck code invalid
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_SAVE_AREA+52(8),__LC_ASYNC_ENTER_TIMER
	mvc	__LC_ASYNC_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA
	tm	__LC_MCCK_CODE+5,0x02	# stored cpu timer value valid?
	bo	BASED(1f)
	la	%r14,__LC_SYNC_ENTER_TIMER
	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
	bl	BASED(0f)
	la	%r14,__LC_ASYNC_ENTER_TIMER
0:	clc	0(8,%r14),__LC_EXIT_TIMER
	bl	BASED(0f)
	la	%r14,__LC_EXIT_TIMER
0:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
	bl	BASED(0f)
	la	%r14,__LC_LAST_UPDATE_TIMER
0:	spt	0(%r14)
	mvc	__LC_ASYNC_ENTER_TIMER(8),0(%r14)
1:
#endif
	tm	__LC_MCCK_CODE+2,0x09	# mwp + ia of old psw valid?
	bno	BASED(mcck_int_main)	# no -> skip cleanup critical
	tm	__LC_MCK_OLD_PSW+1,0x01	# test problem state bit
	bnz	BASED(mcck_int_main)	# from user -> load async stack
	clc	__LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_end)
	bhe	BASED(mcck_int_main)
	clc	__LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_start)
	bl	BASED(mcck_int_main)
	l	%r14,BASED(.Lcleanup_critical)
	basr	%r14,%r14
mcck_int_main:
	l	%r14,__LC_PANIC_STACK	# are we already on the panic stack?
	slr	%r14,%r15
	sra	%r14,PAGE_SHIFT
	be	BASED(0f)
	l	%r15,__LC_PANIC_STACK	# load panic stack
0:	CREATE_STACK_FRAME __LC_MCK_OLD_PSW,__LC_SAVE_AREA+32
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	__LC_MCCK_CODE+2,0x08	# mwp of old psw valid?
	bno	BASED(mcck_no_vtime)	# no -> skip cleanup critical
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	bz	BASED(mcck_no_vtime)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
mcck_no_vtime:
#endif
	l	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	l	%r1,BASED(.Ls390_mcck)
	basr	%r14,%r1		# call machine check handler
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
	bno	BASED(mcck_return)
	l	%r1,__LC_KERNEL_STACK	# switch to kernel stack
	s	%r1,BASED(.Lc_spsize)
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lr	%r15,%r1
	stosm	__SF_EMPTY(%r15),0x04	# turn dat on
	tm	__TI_flags+3(%r9),_TIF_MCCK_PENDING
	bno	BASED(mcck_return)
	TRACE_IRQS_OFF
	l	%r1,BASED(.Ls390_handle_mcck)
	basr	%r14,%r1		# call machine check handler
	TRACE_IRQS_ON
mcck_return:
	mvc	__LC_RETURN_MCCK_PSW(8),SP_PSW(%r15) # move return PSW
	ni	__LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_ASYNC_ENTER_TIMER(8),__LC_SAVE_AREA+52
	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
	bno	BASED(0f)
	lm	%r0,%r15,SP_R0(%r15)	# load gprs 0-15
	stpt	__LC_EXIT_TIMER
	lpsw	__LC_RETURN_MCCK_PSW	# back to caller
0:
#endif
	lm	%r0,%r15,SP_R0(%r15)	# load gprs 0-15
	lpsw	__LC_RETURN_MCCK_PSW	# back to caller

	RESTORE_ALL __LC_RETURN_MCCK_PSW,0

/*
 * Restart interruption handler, kick starter for additional CPUs
 */
#ifdef CONFIG_SMP
	__CPUINIT
	.globl restart_int_handler
restart_int_handler:
	l	%r15,__LC_SAVE_AREA+60	# load ksp
	lctl	%c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs
	lam	%a0,%a15,__LC_AREGS_SAVE_AREA
	lm	%r6,%r15,__SF_GPRS(%r15) # load registers from clone
	stosm	__SF_EMPTY(%r15),0x04	# now we can turn dat on
	basr	%r14,0
	l	%r14,restart_addr-.(%r14)
	br	%r14			# branch to start_secondary
restart_addr:
	.long	start_secondary
	.previous
#else
/*
 * If we do not run with SMP enabled, let the new CPU crash ...
 */
	.globl restart_int_handler
restart_int_handler:
	basr	%r1,0
restart_base:
	lpsw	restart_crash-restart_base(%r1)
	.align	8
restart_crash:
	.long	0x000a0000,0x00000000
restart_go:
#endif

#ifdef CONFIG_CHECK_STACK
/*
 * The synchronous or the asynchronous stack overflowed. We are dead.
 * No need to properly save the registers, we are going to panic anyway.
 * Setup a pt_regs so that show_trace can provide a good call trace.
 */
stack_overflow:
	l	%r15,__LC_PANIC_STACK	# change to panic stack
	sl	%r15,BASED(.Lc_spsize)
	mvc	SP_PSW(8,%r15),0(%r12)	# move user PSW to stack
	stm	%r0,%r11,SP_R0(%r15)	# store gprs %r0-%r11 to kernel stack
	la	%r1,__LC_SAVE_AREA
	ch	%r12,BASED(.L0x020)	# old psw addr == __LC_SVC_OLD_PSW ?
	be	BASED(0f)
	ch	%r12,BASED(.L0x028)	# old psw addr == __LC_PGM_OLD_PSW ?
	be	BASED(0f)
	la	%r1,__LC_SAVE_AREA+16
0:	mvc	SP_R12(16,%r15),0(%r1)	# move %r12-%r15 to stack
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # clear back chain
	l	%r1,BASED(1f)		# branch to kernel_stack_overflow
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	br	%r1
1:	.long	kernel_stack_overflow
#endif

cleanup_table_system_call:
	.long	system_call + 0x80000000, sysc_do_svc + 0x80000000
cleanup_table_sysc_return:
	.long	sysc_return + 0x80000000, sysc_leave + 0x80000000
cleanup_table_sysc_leave:
	.long	sysc_leave + 0x80000000, sysc_done + 0x80000000
cleanup_table_sysc_work_loop:
	.long	sysc_work_loop + 0x80000000, sysc_work_done + 0x80000000
cleanup_table_io_return:
	.long	io_return + 0x80000000, io_leave + 0x80000000
cleanup_table_io_leave:
	.long	io_leave + 0x80000000, io_done + 0x80000000
cleanup_table_io_work_loop:
	.long	io_work_loop + 0x80000000, io_work_done + 0x80000000

cleanup_critical:
	clc	4(4,%r12),BASED(cleanup_table_system_call)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_system_call+4)
	bl	BASED(cleanup_system_call)
0:
	clc	4(4,%r12),BASED(cleanup_table_sysc_return)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_sysc_return+4)
	bl	BASED(cleanup_sysc_return)
0:
	clc	4(4,%r12),BASED(cleanup_table_sysc_leave)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_sysc_leave+4)
	bl	BASED(cleanup_sysc_leave)
0:
	clc	4(4,%r12),BASED(cleanup_table_sysc_work_loop)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_sysc_work_loop+4)
	bl	BASED(cleanup_sysc_return)
0:
	clc	4(4,%r12),BASED(cleanup_table_io_return)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_io_return+4)
	bl	BASED(cleanup_io_return)
0:
	clc	4(4,%r12),BASED(cleanup_table_io_leave)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_io_leave+4)
	bl	BASED(cleanup_io_leave)
0:
	clc	4(4,%r12),BASED(cleanup_table_io_work_loop)
	bl	BASED(0f)
	clc	4(4,%r12),BASED(cleanup_table_io_work_loop+4)
	bl	BASED(cleanup_io_return)
0:
	br	%r14

cleanup_system_call:
	mvc	__LC_RETURN_PSW(8),0(%r12)
	c	%r12,BASED(.Lmck_old_psw)
	be	BASED(0f)
	la	%r12,__LC_SAVE_AREA+16
	b	BASED(1f)
0:	la	%r12,__LC_SAVE_AREA+32
1:
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	clc	__LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+4)
	bh	BASED(0f)
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
0:	clc	__LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+8)
	bhe	BASED(cleanup_vtime)
#endif
	clc	__LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn)
	bh	BASED(0f)
	mvc	__LC_SAVE_AREA(16),0(%r12)
0:	st	%r13,4(%r12)
	st	%r12,__LC_SAVE_AREA+48	# argh
	SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	l	%r12,__LC_SAVE_AREA+48	# argh
	st	%r15,12(%r12)
	lh	%r7,0x8a
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
cleanup_vtime:
	clc	__LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+12)
	bhe	BASED(cleanup_stime)
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
cleanup_stime:
	clc	__LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+16)
	bh	BASED(cleanup_update)
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
cleanup_update:
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
#endif
	mvc	__LC_RETURN_PSW+4(4),BASED(cleanup_table_system_call+4)
	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_system_call_insn:
	.long	sysc_saveall + 0x80000000
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.long	system_call + 0x80000000
	.long	sysc_vtime + 0x80000000
	.long	sysc_stime + 0x80000000
	.long	sysc_update + 0x80000000
#endif

cleanup_sysc_return:
	mvc	__LC_RETURN_PSW(4),0(%r12)
	mvc	__LC_RETURN_PSW+4(4),BASED(cleanup_table_sysc_return)
	la	%r12,__LC_RETURN_PSW
	br	%r14

cleanup_sysc_leave:
	clc	4(4,%r12),BASED(cleanup_sysc_leave_insn)
	be	BASED(2f)
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
	clc	4(4,%r12),BASED(cleanup_sysc_leave_insn+4)
	be	BASED(2f)
#endif
	mvc	__LC_RETURN_PSW(8),SP_PSW(%r15)
	c	%r12,BASED(.Lmck_old_psw)
	bne	BASED(0f)
	mvc	__LC_SAVE_AREA+32(16),SP_R12(%r15)
	b	BASED(1f)
0:	mvc	__LC_SAVE_AREA+16(16),SP_R12(%r15)
1:	lm	%r0,%r11,SP_R0(%r15)
	l	%r15,SP_R15(%r15)
2:	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_sysc_leave_insn:
	.long	sysc_done - 4 + 0x80000000
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.long	sysc_done - 8 + 0x80000000
#endif

cleanup_io_return:
	mvc	__LC_RETURN_PSW(4),0(%r12)
	mvc	__LC_RETURN_PSW+4(4),BASED(cleanup_table_io_work_loop)
	la	%r12,__LC_RETURN_PSW
	br	%r14

cleanup_io_leave:
	clc	4(4,%r12),BASED(cleanup_io_leave_insn)
	be	BASED(2f)
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
	clc	4(4,%r12),BASED(cleanup_io_leave_insn+4)
	be	BASED(2f)
#endif
	mvc	__LC_RETURN_PSW(8),SP_PSW(%r15)
	c	%r12,BASED(.Lmck_old_psw)
	bne	BASED(0f)
	mvc	__LC_SAVE_AREA+32(16),SP_R12(%r15)
	b	BASED(1f)
0:	mvc	__LC_SAVE_AREA+16(16),SP_R12(%r15)
1:	lm	%r0,%r11,SP_R0(%r15)
	l	%r15,SP_R15(%r15)
2:	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_io_leave_insn:
	.long	io_done - 4 + 0x80000000
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.long	io_done - 8 + 0x80000000
#endif

/*
 * Integer constants
 */
		.align	4
.Lc_spsize:	.long	SP_SIZE
.Lc_overhead:	.long	STACK_FRAME_OVERHEAD
.Lnr_syscalls:	.long	NR_syscalls
.L0x018:	.short	0x018
.L0x020:	.short	0x020
.L0x028:	.short	0x028
.L0x030:	.short	0x030
.L0x038:	.short	0x038
.Lc_1:		.long	1

/*
 * Symbol constants
 */
.Ls390_mcck:	.long	s390_do_machine_check
.Ls390_handle_mcck:
		.long	s390_handle_mcck
.Lmck_old_psw:	.long	__LC_MCK_OLD_PSW
.Ldo_IRQ:	.long	do_IRQ
.Ldo_extint:	.long	do_extint
.Ldo_signal:	.long	do_signal
.Lhandle_per:	.long	do_single_step
.Ldo_execve:	.long	do_execve
.Lexecve_tail:	.long	execve_tail
.Ljump_table:	.long	pgm_check_table
.Lschedule:	.long	schedule
#ifdef CONFIG_PREEMPT
.Lpreempt_schedule_irq:
		.long	preempt_schedule_irq
#endif
.Ltrace:	.long	syscall_trace
.Lschedtail:	.long	schedule_tail
.Lsysc_table:	.long	sys_call_table
#ifdef CONFIG_TRACE_IRQFLAGS
.Ltrace_irq_on: .long	trace_hardirqs_on
.Ltrace_irq_off:
		.long	trace_hardirqs_off
.Llockdep_sys_exit:
		.long	lockdep_sys_exit
#endif
.Lcritical_start:
		.long	__critical_start + 0x80000000
.Lcritical_end:
		.long	__critical_end + 0x80000000
.Lcleanup_critical:
		.long	cleanup_critical

		.section .rodata, "a"
#define SYSCALL(esa,esame,emu)	.long esa
sys_call_table:
#include "syscalls.S"
#undef SYSCALL