Linux Audio

Check our new training course

Embedded Linux Audio

Check our new training course
with Creative Commons CC-BY-SA
lecture materials

Bootlin logo

Elixir Cross Referencer

Loading...
   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
/*
 *  arch/s390/kernel/entry64.S
 *    S390 low-level entry points.
 *
 *    Copyright (C) IBM Corp. 1999,2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
 *		 Hartmut Penner (hp@de.ibm.com),
 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
 */

#include <linux/sys.h>
#include <linux/linkage.h>
#include <asm/cache.h>
#include <asm/lowcore.h>
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
#include <asm/page.h>

/*
 * Stack layout for the system_call stack entry.
 * The first few entries are identical to the user_regs_struct.
 */
SP_PTREGS    =	STACK_FRAME_OVERHEAD
SP_ARGS      =	STACK_FRAME_OVERHEAD + __PT_ARGS
SP_PSW	     =	STACK_FRAME_OVERHEAD + __PT_PSW
SP_R0	     =	STACK_FRAME_OVERHEAD + __PT_GPRS
SP_R1	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 8
SP_R2	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 16
SP_R3	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 24
SP_R4	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 32
SP_R5	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 40
SP_R6	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 48
SP_R7	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 56
SP_R8	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 64
SP_R9	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 72
SP_R10	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 80
SP_R11	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 88
SP_R12	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 96
SP_R13	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 104
SP_R14	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 112
SP_R15	     =	STACK_FRAME_OVERHEAD + __PT_GPRS + 120
SP_ORIG_R2   =	STACK_FRAME_OVERHEAD + __PT_ORIG_GPR2
SP_ILC	     =	STACK_FRAME_OVERHEAD + __PT_ILC
SP_TRAP      =	STACK_FRAME_OVERHEAD + __PT_TRAP
SP_SIZE      =	STACK_FRAME_OVERHEAD + __PT_SIZE

STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
STACK_SIZE  = 1 << STACK_SHIFT

_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \
		 _TIF_MCCK_PENDING | _TIF_RESTART_SVC | _TIF_SINGLE_STEP )
_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \
		 _TIF_MCCK_PENDING)

#define BASED(name) name-system_call(%r13)

#ifdef CONFIG_TRACE_IRQFLAGS
	.macro	TRACE_IRQS_ON
	 brasl	%r14,trace_hardirqs_on
	.endm

	.macro	TRACE_IRQS_OFF
	 brasl	%r14,trace_hardirqs_off
	.endm
#else
#define TRACE_IRQS_ON
#define TRACE_IRQS_OFF
#endif

	.macro	STORE_TIMER lc_offset
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	stpt	\lc_offset
#endif
	.endm

#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.macro	UPDATE_VTIME lc_from,lc_to,lc_sum
	lg	%r10,\lc_from
	slg	%r10,\lc_to
	alg	%r10,\lc_sum
	stg	%r10,\lc_sum
	.endm
#endif

/*
 * Register usage in interrupt handlers:
 *    R9  - pointer to current task structure
 *    R13 - pointer to literal pool
 *    R14 - return register for function calls
 *    R15 - kernel stack pointer
 */

	.macro	SAVE_ALL_BASE savearea
	stmg	%r12,%r15,\savearea
	larl	%r13,system_call
	.endm

	.macro	SAVE_ALL_SYNC psworg,savearea
	la	%r12,\psworg
	tm	\psworg+1,0x01		# test problem state bit
	jz	2f			# skip stack setup save
	lg	%r15,__LC_KERNEL_STACK	# problem state -> load ksp
#ifdef CONFIG_CHECK_STACK
	j	3f
2:	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
	jz	stack_overflow
3:
#endif
2:
	.endm

	.macro	SAVE_ALL_ASYNC psworg,savearea
	la	%r12,\psworg
	tm	\psworg+1,0x01		# test problem state bit
	jnz	1f			# from user -> load kernel stack
	clc	\psworg+8(8),BASED(.Lcritical_end)
	jhe	0f
	clc	\psworg+8(8),BASED(.Lcritical_start)
	jl	0f
	brasl	%r14,cleanup_critical
	tm	1(%r12),0x01		# retest problem state after cleanup
	jnz	1f
0:	lg	%r14,__LC_ASYNC_STACK	# are we already on the async. stack ?
	slgr	%r14,%r15
	srag	%r14,%r14,STACK_SHIFT
	jz	2f
1:	lg	%r15,__LC_ASYNC_STACK	# load async stack
#ifdef CONFIG_CHECK_STACK
	j	3f
2:	tml	%r15,STACK_SIZE - CONFIG_STACK_GUARD
	jz	stack_overflow
3:
#endif
2:
	.endm

	.macro	CREATE_STACK_FRAME psworg,savearea
	aghi	%r15,-SP_SIZE		# make room for registers & psw
	mvc	SP_PSW(16,%r15),0(%r12)	# move user PSW to stack
	la	%r12,\psworg
	stg	%r2,SP_ORIG_R2(%r15)	# store original content of gpr 2
	icm	%r12,12,__LC_SVC_ILC
	stmg	%r0,%r11,SP_R0(%r15)	# store gprs %r0-%r11 to kernel stack
	st	%r12,SP_ILC(%r15)
	mvc	SP_R12(32,%r15),\savearea # move %r12-%r15 to stack
	la	%r12,0
	stg	%r12,__SF_BACKCHAIN(%r15)
	.endm

	.macro	RESTORE_ALL psworg,sync
	mvc	\psworg(16),SP_PSW(%r15) # move user PSW to lowcore
	.if !\sync
	ni	\psworg+1,0xfd		# clear wait state bit
	.endif
	lmg	%r0,%r15,SP_R0(%r15)	# load gprs 0-15 of user
	STORE_TIMER __LC_EXIT_TIMER
	lpswe	\psworg			# back to caller
	.endm

/*
 * Scheduler resume function, called by switch_to
 *  gpr2 = (task_struct *) prev
 *  gpr3 = (task_struct *) next
 * Returns:
 *  gpr2 = prev
 */
	.globl	__switch_to
__switch_to:
	tm	__THREAD_per+4(%r3),0xe8 # is the new process using per ?
	jz	__switch_to_noper		# if not we're fine
	stctg	%c9,%c11,__SF_EMPTY(%r15)# We are using per stuff
	clc	__THREAD_per(24,%r3),__SF_EMPTY(%r15)
	je	__switch_to_noper	     # we got away without bashing TLB's
	lctlg	%c9,%c11,__THREAD_per(%r3)	# Nope we didn't
__switch_to_noper:
	lg	%r4,__THREAD_info(%r2)		    # get thread_info of prev
	tm	__TI_flags+7(%r4),_TIF_MCCK_PENDING # machine check pending?
	jz	__switch_to_no_mcck
	ni	__TI_flags+7(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
	lg	%r4,__THREAD_info(%r3)		    # get thread_info of next
	oi	__TI_flags+7(%r4),_TIF_MCCK_PENDING # set it in next
__switch_to_no_mcck:
	stmg	%r6,%r15,__SF_GPRS(%r15)# store __switch_to registers of prev task
	stg	%r15,__THREAD_ksp(%r2)	# store kernel stack to prev->tss.ksp
	lg	%r15,__THREAD_ksp(%r3)	# load kernel stack from next->tss.ksp
	lmg	%r6,%r15,__SF_GPRS(%r15)# load __switch_to registers of next task
	stg	%r3,__LC_CURRENT	# __LC_CURRENT = current task struct
	lctl	%c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
	lg	%r3,__THREAD_info(%r3)	# load thread_info from task struct
	stg	%r3,__LC_THREAD_INFO
	aghi	%r3,STACK_SIZE
	stg	%r3,__LC_KERNEL_STACK	# __LC_KERNEL_STACK = new kernel stack
	br	%r14

__critical_start:
/*
 * SVC interrupt handler routine. System calls are synchronous events and
 * are executed with interrupts enabled.
 */

	.globl	system_call
system_call:
	STORE_TIMER __LC_SYNC_ENTER_TIMER
sysc_saveall:
	SAVE_ALL_BASE __LC_SAVE_AREA
	SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	llgh	%r7,__LC_SVC_INT_CODE	# get svc number from lowcore
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
sysc_vtime:
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	sysc_do_svc
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
sysc_stime:
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
sysc_update:
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
#endif
sysc_do_svc:
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	slag	%r7,%r7,2	# *4 and test for svc 0
	jnz	sysc_nr_ok
	# svc 0: system call number in %r1
	cl	%r1,BASED(.Lnr_syscalls)
	jnl	sysc_nr_ok
	lgfr	%r7,%r1 	# clear high word in r1
	slag	%r7,%r7,2	# svc 0: system call number in %r1
sysc_nr_ok:
	mvc	SP_ARGS(8,%r15),SP_R7(%r15)
sysc_do_restart:
	larl	%r10,sys_call_table
#ifdef CONFIG_COMPAT
	tm	__TI_flags+5(%r9),(_TIF_31BIT>>16)  # running in 31 bit mode ?
	jno	sysc_noemu
	larl	%r10,sys_call_table_emu  # use 31 bit emulation system calls
sysc_noemu:
#endif
	tm	__TI_flags+7(%r9),(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)
	lgf	%r8,0(%r7,%r10) # load address of system call routine
	jnz	sysc_tracesys
	basr	%r14,%r8	# call sys_xxxx
	stg	%r2,SP_R2(%r15) # store return value (change R2 on stack)
				# ATTENTION: check sys_execve_glue before
				# changing anything here !!

sysc_return:
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
	jno	sysc_leave
	tm	__TI_flags+7(%r9),_TIF_WORK_SVC
	jnz	sysc_work	# there is work to do (signals etc.)
sysc_leave:
	RESTORE_ALL __LC_RETURN_PSW,1

#
# recheck if there is more work to do
#
sysc_work_loop:
	tm	__TI_flags+7(%r9),_TIF_WORK_SVC
	jz	sysc_leave	  # there is no work to do
#
# One of the work bits is on. Find out which one.
#
sysc_work:
	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
	jo	sysc_mcck_pending
	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
	jo	sysc_reschedule
	tm	__TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)
	jnz	sysc_sigpending
	tm	__TI_flags+7(%r9),_TIF_RESTART_SVC
	jo	sysc_restart
	tm	__TI_flags+7(%r9),_TIF_SINGLE_STEP
	jo	sysc_singlestep
	j	sysc_leave

#
# _TIF_NEED_RESCHED is set, call schedule
#
sysc_reschedule:
	larl	%r14,sysc_work_loop
	jg	schedule	# return point is sysc_return

#
# _TIF_MCCK_PENDING is set, call handler
#
sysc_mcck_pending:
	larl	%r14,sysc_work_loop
	jg	s390_handle_mcck	# TIF bit will be cleared by handler

#
# _TIF_SIGPENDING or _TIF_RESTORE_SIGMASK is set, call do_signal
#
sysc_sigpending:
	ni	__TI_flags+7(%r9),255-_TIF_SINGLE_STEP # clear TIF_SINGLE_STEP
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	brasl	%r14,do_signal		# call do_signal
	tm	__TI_flags+7(%r9),_TIF_RESTART_SVC
	jo	sysc_restart
	tm	__TI_flags+7(%r9),_TIF_SINGLE_STEP
	jo	sysc_singlestep
	j	sysc_work_loop

#
# _TIF_RESTART_SVC is set, set up registers and restart svc
#
sysc_restart:
	ni	__TI_flags+7(%r9),255-_TIF_RESTART_SVC # clear TIF_RESTART_SVC
	lg	%r7,SP_R2(%r15)		# load new svc number
	slag	%r7,%r7,2		# *4
	mvc	SP_R2(8,%r15),SP_ORIG_R2(%r15) # restore first argument
	lmg	%r2,%r6,SP_R2(%r15)	# load svc arguments
	j	sysc_do_restart 	# restart svc

#
# _TIF_SINGLE_STEP is set, call do_single_step
#
sysc_singlestep:
	ni	__TI_flags+7(%r9),255-_TIF_SINGLE_STEP # clear TIF_SINGLE_STEP
	lhi	%r0,__LC_PGM_OLD_PSW
	sth	%r0,SP_TRAP(%r15)	# set trap indication to pgm check
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	larl	%r14,sysc_return	# load adr. of system return
	jg	do_single_step		# branch to do_sigtrap

#
# call syscall_trace before and after system call
# special linkage: %r12 contains the return address for trace_svc
#
sysc_tracesys:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	la	%r3,0
	srl	%r7,2
	stg	%r7,SP_R2(%r15)
	brasl	%r14,syscall_trace
	lghi	%r0,NR_syscalls
	clg	%r0,SP_R2(%r15)
	jnh	sysc_tracenogo
	lg	%r7,SP_R2(%r15)		# strace might have changed the
	sll	%r7,2			# system call
	lgf	%r8,0(%r7,%r10)
sysc_tracego:
	lmg	%r3,%r6,SP_R3(%r15)
	lg	%r2,SP_ORIG_R2(%r15)
	basr	%r14,%r8		# call sys_xxx
	stg	%r2,SP_R2(%r15)		# store return value
sysc_tracenogo:
	tm	__TI_flags+7(%r9),(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)
	jz	sysc_return
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	la	%r3,1
	larl	%r14,sysc_return	# return point is sysc_return
	jg	syscall_trace

#
# a new process exits the kernel with ret_from_fork
#
	.globl	ret_from_fork
ret_from_fork:
	lg	%r13,__LC_SVC_NEW_PSW+8
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	tm	SP_PSW+1(%r15),0x01	# forking a kernel thread ?
	jo	0f
	stg	%r15,SP_R15(%r15)	# store stack pointer for new kthread
0:	brasl	%r14,schedule_tail
	TRACE_IRQS_ON
	stosm	24(%r15),0x03		# reenable interrupts
	j	sysc_return

#
# clone, fork, vfork, exec and sigreturn need glue,
# because they all expect pt_regs as parameter,
# but are called with different parameter.
# return-address is set up above
#
sys_clone_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	jg	sys_clone		# branch to sys_clone

#ifdef CONFIG_COMPAT
sys32_clone_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	jg	sys32_clone		# branch to sys32_clone
#endif

sys_fork_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	jg	sys_fork		# branch to sys_fork

sys_vfork_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	jg	sys_vfork		# branch to sys_vfork

sys_execve_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	lgr	%r12,%r14		# save return address
	brasl	%r14,sys_execve 	# call sys_execve
	ltgr	%r2,%r2 		# check if execve failed
	bnz	0(%r12) 		# it did fail -> store result in gpr2
	b	6(%r12) 		# SKIP STG 2,SP_R2(15) in
					# system_call/sysc_tracesys
#ifdef CONFIG_COMPAT
sys32_execve_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	lgr	%r12,%r14		# save return address
	brasl	%r14,sys32_execve	# call sys32_execve
	ltgr	%r2,%r2 		# check if execve failed
	bnz	0(%r12) 		# it did fail -> store result in gpr2
	b	6(%r12) 		# SKIP STG 2,SP_R2(15) in
					# system_call/sysc_tracesys
#endif

sys_sigreturn_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys_sigreturn		# branch to sys_sigreturn

#ifdef CONFIG_COMPAT
sys32_sigreturn_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys32_sigreturn 	# branch to sys32_sigreturn
#endif

sys_rt_sigreturn_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys_rt_sigreturn	# branch to sys_sigreturn

#ifdef CONFIG_COMPAT
sys32_rt_sigreturn_glue:
	la	%r2,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys32_rt_sigreturn	# branch to sys32_sigreturn
#endif

sys_sigaltstack_glue:
	la	%r4,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys_sigaltstack 	# branch to sys_sigreturn

#ifdef CONFIG_COMPAT
sys32_sigaltstack_glue:
	la	%r4,SP_PTREGS(%r15)	# load pt_regs as parameter
	jg	sys32_sigaltstack_wrapper # branch to sys_sigreturn
#endif

/*
 * Program check handler routine
 */

	.globl	pgm_check_handler
pgm_check_handler:
/*
 * First we need to check for a special case:
 * Single stepping an instruction that disables the PER event mask will
 * cause a PER event AFTER the mask has been set. Example: SVC or LPSW.
 * For a single stepped SVC the program check handler gets control after
 * the SVC new PSW has been loaded. But we want to execute the SVC first and
 * then handle the PER event. Therefore we update the SVC old PSW to point
 * to the pgm_check_handler and branch to the SVC handler after we checked
 * if we have to load the kernel stack register.
 * For every other possible cause for PER event without the PER mask set
 * we just ignore the PER event (FIXME: is there anything we have to do
 * for LPSW?).
 */
	STORE_TIMER __LC_SYNC_ENTER_TIMER
	SAVE_ALL_BASE __LC_SAVE_AREA
	tm	__LC_PGM_INT_CODE+1,0x80 # check whether we got a per exception
	jnz	pgm_per 		 # got per exception -> special case
	SAVE_ALL_SYNC __LC_PGM_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_PGM_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	pgm_no_vtime
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
pgm_no_vtime:
#endif
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	lgf	%r3,__LC_PGM_ILC	# load program interruption code
	lghi	%r8,0x7f
	ngr	%r8,%r3
pgm_do_call:
	sll	%r8,3
	larl	%r1,pgm_check_table
	lg	%r1,0(%r8,%r1)		# load address of handler routine
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	larl	%r14,sysc_return
	br	%r1			# branch to interrupt-handler

#
# handle per exception
#
pgm_per:
	tm	__LC_PGM_OLD_PSW,0x40	# test if per event recording is on
	jnz	pgm_per_std		# ok, normal per event from user space
# ok its one of the special cases, now we need to find out which one
	clc	__LC_PGM_OLD_PSW(16),__LC_SVC_NEW_PSW
	je	pgm_svcper
# no interesting special case, ignore PER event
	lmg	%r12,%r15,__LC_SAVE_AREA
	lpswe	__LC_PGM_OLD_PSW

#
# Normal per exception
#
pgm_per_std:
	SAVE_ALL_SYNC __LC_PGM_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_PGM_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	pgm_no_vtime2
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
pgm_no_vtime2:
#endif
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	lg	%r1,__TI_task(%r9)
	tm	SP_PSW+1(%r15),0x01	# kernel per event ?
	jz	kernel_per
	mvc	__THREAD_per+__PER_atmid(2,%r1),__LC_PER_ATMID
	mvc	__THREAD_per+__PER_address(8,%r1),__LC_PER_ADDRESS
	mvc	__THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID
	oi	__TI_flags+7(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP
	lgf	%r3,__LC_PGM_ILC	# load program interruption code
	lghi	%r8,0x7f
	ngr	%r8,%r3			# clear per-event-bit and ilc
	je	sysc_return
	j	pgm_do_call

#
# it was a single stepped SVC that is causing all the trouble
#
pgm_svcper:
	SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	pgm_no_vtime3
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
pgm_no_vtime3:
#endif
	llgh	%r7,__LC_SVC_INT_CODE	# get svc number from lowcore
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	lg	%r1,__TI_task(%r9)
	mvc	__THREAD_per+__PER_atmid(2,%r1),__LC_PER_ATMID
	mvc	__THREAD_per+__PER_address(8,%r1),__LC_PER_ADDRESS
	mvc	__THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID
	oi	__TI_flags+7(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP
	TRACE_IRQS_ON
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	j	sysc_do_svc

#
# per was called from kernel, must be kprobes
#
kernel_per:
	lhi	%r0,__LC_PGM_OLD_PSW
	sth	%r0,SP_TRAP(%r15)	# set trap indication to pgm check
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	larl	%r14,sysc_leave		# load adr. of system ret, no work
	jg	do_single_step		# branch to do_single_step

/*
 * IO interrupt handler routine
 */
	.globl io_int_handler
io_int_handler:
	STORE_TIMER __LC_ASYNC_ENTER_TIMER
	stck	__LC_INT_CLOCK
	SAVE_ALL_BASE __LC_SAVE_AREA+32
	SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+32
	CREATE_STACK_FRAME __LC_IO_OLD_PSW,__LC_SAVE_AREA+32
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	io_no_vtime
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
io_no_vtime:
#endif
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	brasl	%r14,do_IRQ		# call standard irq handler
	TRACE_IRQS_ON

io_return:
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
#ifdef CONFIG_PREEMPT
	jno	io_preempt		# no -> check for preemptive scheduling
#else
	jno	io_leave		# no-> skip resched & signal
#endif
	tm	__TI_flags+7(%r9),_TIF_WORK_INT
	jnz	io_work 		# there is work to do (signals etc.)
io_leave:
	RESTORE_ALL __LC_RETURN_PSW,0
io_done:

#ifdef CONFIG_PREEMPT
io_preempt:
	icm	%r0,15,__TI_precount(%r9)
	jnz	io_leave
	# switch to kernel stack
	lg	%r1,SP_R15(%r15)
	aghi	%r1,-SP_SIZE
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lgr	%r15,%r1
io_resume_loop:
	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
	jno	io_leave
	larl	%r1,.Lc_pactive
	mvc	__TI_precount(4,%r9),0(%r1)
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	brasl	%r14,schedule		# call schedule
	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
	xc	__TI_precount(4,%r9),__TI_precount(%r9)
	j	io_resume_loop
#endif

#
# switch to kernel stack, then check TIF bits
#
io_work:
	lg	%r1,__LC_KERNEL_STACK
	aghi	%r1,-SP_SIZE
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lgr	%r15,%r1
#
# One of the work bits is on. Find out which one.
# Checked are: _TIF_SIGPENDING, _TIF_RESTORE_SIGPENDING, _TIF_NEED_RESCHED
#	       and _TIF_MCCK_PENDING
#
io_work_loop:
	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
	jo	io_mcck_pending
	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
	jo	io_reschedule
	tm	__TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)
	jnz	io_sigpending
	j	io_leave

#
# _TIF_MCCK_PENDING is set, call handler
#
io_mcck_pending:
	larl	%r14,io_work_loop
	jg	s390_handle_mcck	# TIF bit will be cleared by handler

#
# _TIF_NEED_RESCHED is set, call schedule
#
io_reschedule:
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	brasl	%r14,schedule		# call scheduler
	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
	tm	__TI_flags+7(%r9),_TIF_WORK_INT
	jz	io_leave		# there is no work to do
	j	io_work_loop

#
# _TIF_SIGPENDING or _TIF_RESTORE_SIGMASK is set, call do_signal
#
io_sigpending:
	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	brasl	%r14,do_signal		# call do_signal
	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
	j	io_work_loop

/*
 * External interrupt handler routine
 */
	.globl	ext_int_handler
ext_int_handler:
	STORE_TIMER __LC_ASYNC_ENTER_TIMER
	stck	__LC_INT_CLOCK
	SAVE_ALL_BASE __LC_SAVE_AREA+32
	SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+32
	CREATE_STACK_FRAME __LC_EXT_OLD_PSW,__LC_SAVE_AREA+32
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	ext_no_vtime
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
ext_no_vtime:
#endif
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	TRACE_IRQS_OFF
	la	%r2,SP_PTREGS(%r15)	# address of register-save area
	llgh	%r3,__LC_EXT_INT_CODE	# get interruption code
	brasl	%r14,do_extint
	TRACE_IRQS_ON
	j	io_return

__critical_end:

/*
 * Machine check handler routines
 */
	.globl mcck_int_handler
mcck_int_handler:
	la	%r1,4095		# revalidate r1
	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# revalidate cpu timer
	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
	SAVE_ALL_BASE __LC_SAVE_AREA+64
	la	%r12,__LC_MCK_OLD_PSW
	tm	__LC_MCCK_CODE,0x80	# system damage?
	jo	mcck_int_main		# yes -> rest of mcck code invalid
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	la	%r14,4095
	mvc	__LC_SAVE_AREA+104(8),__LC_ASYNC_ENTER_TIMER
	mvc	__LC_ASYNC_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA-4095(%r14)
	tm	__LC_MCCK_CODE+5,0x02	# stored cpu timer value valid?
	jo	1f
	la	%r14,__LC_SYNC_ENTER_TIMER
	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
	jl	0f
	la	%r14,__LC_ASYNC_ENTER_TIMER
0:	clc	0(8,%r14),__LC_EXIT_TIMER
	jl	0f
	la	%r14,__LC_EXIT_TIMER
0:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
	jl	0f
	la	%r14,__LC_LAST_UPDATE_TIMER
0:	spt	0(%r14)
	mvc	__LC_ASYNC_ENTER_TIMER(8),0(%r14)
1:
#endif
	tm	__LC_MCCK_CODE+2,0x09	# mwp + ia of old psw valid?
	jno	mcck_int_main		# no -> skip cleanup critical
	tm	__LC_MCK_OLD_PSW+1,0x01 # test problem state bit
	jnz	mcck_int_main		# from user -> load kernel stack
	clc	__LC_MCK_OLD_PSW+8(8),BASED(.Lcritical_end)
	jhe	mcck_int_main
	clc	__LC_MCK_OLD_PSW+8(8),BASED(.Lcritical_start)
	jl	mcck_int_main
	brasl	%r14,cleanup_critical
mcck_int_main:
	lg	%r14,__LC_PANIC_STACK	# are we already on the panic stack?
	slgr	%r14,%r15
	srag	%r14,%r14,PAGE_SHIFT
	jz	0f
	lg	%r15,__LC_PANIC_STACK	# load panic stack
0:	CREATE_STACK_FRAME __LC_MCK_OLD_PSW,__LC_SAVE_AREA+64
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	tm	__LC_MCCK_CODE+2,0x08	# mwp of old psw valid?
	jno	mcck_no_vtime		# no -> no timer update
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	mcck_no_vtime
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
mcck_no_vtime:
#endif
	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	brasl	%r14,s390_do_machine_check
	tm	SP_PSW+1(%r15),0x01	# returning to user ?
	jno	mcck_return
	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
	aghi	%r1,-SP_SIZE
	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
	lgr	%r15,%r1
	stosm	__SF_EMPTY(%r15),0x04	# turn dat on
	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
	jno	mcck_return
	TRACE_IRQS_OFF
	brasl	%r14,s390_handle_mcck
	TRACE_IRQS_ON
mcck_return:
	mvc	__LC_RETURN_MCCK_PSW(16),SP_PSW(%r15) # move return PSW
	ni	__LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
	lmg	%r0,%r15,SP_R0(%r15)	# load gprs 0-15
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_ASYNC_ENTER_TIMER(8),__LC_SAVE_AREA+104
	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
	jno	0f
	stpt	__LC_EXIT_TIMER
0:
#endif
	lpswe	__LC_RETURN_MCCK_PSW	# back to caller

#ifdef CONFIG_SMP
/*
 * Restart interruption handler, kick starter for additional CPUs
 */
	.globl restart_int_handler
restart_int_handler:
	lg	%r15,__LC_SAVE_AREA+120 # load ksp
	lghi	%r10,__LC_CREGS_SAVE_AREA
	lctlg	%c0,%c15,0(%r10) # get new ctl regs
	lghi	%r10,__LC_AREGS_SAVE_AREA
	lam	%a0,%a15,0(%r10)
	lmg	%r6,%r15,__SF_GPRS(%r15) # load registers from clone
	stosm	__SF_EMPTY(%r15),0x04	# now we can turn dat on
	jg	start_secondary
#else
/*
 * If we do not run with SMP enabled, let the new CPU crash ...
 */
	.globl restart_int_handler
restart_int_handler:
	basr	%r1,0
restart_base:
	lpswe	restart_crash-restart_base(%r1)
	.align 8
restart_crash:
	.long  0x000a0000,0x00000000,0x00000000,0x00000000
restart_go:
#endif

#ifdef CONFIG_CHECK_STACK
/*
 * The synchronous or the asynchronous stack overflowed. We are dead.
 * No need to properly save the registers, we are going to panic anyway.
 * Setup a pt_regs so that show_trace can provide a good call trace.
 */
stack_overflow:
	lg	%r15,__LC_PANIC_STACK	# change to panic stack
	aghi	%r15,-SP_SIZE
	mvc	SP_PSW(16,%r15),0(%r12)	# move user PSW to stack
	stmg	%r0,%r11,SP_R0(%r15)	# store gprs %r0-%r11 to kernel stack
	la	%r1,__LC_SAVE_AREA
	chi	%r12,__LC_SVC_OLD_PSW
	je	0f
	chi	%r12,__LC_PGM_OLD_PSW
	je	0f
	la	%r1,__LC_SAVE_AREA+32
0:	mvc	SP_R12(32,%r15),0(%r1)	# move %r12-%r15 to stack
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) # clear back chain
	la	%r2,SP_PTREGS(%r15)	# load pt_regs
	jg	kernel_stack_overflow
#endif

cleanup_table_system_call:
	.quad	system_call, sysc_do_svc
cleanup_table_sysc_return:
	.quad	sysc_return, sysc_leave
cleanup_table_sysc_leave:
	.quad	sysc_leave, sysc_work_loop
cleanup_table_sysc_work_loop:
	.quad	sysc_work_loop, sysc_reschedule
cleanup_table_io_return:
	.quad	io_return, io_leave
cleanup_table_io_leave:
	.quad	io_leave, io_done
cleanup_table_io_work_loop:
	.quad	io_work_loop, io_mcck_pending

cleanup_critical:
	clc	8(8,%r12),BASED(cleanup_table_system_call)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_system_call+8)
	jl	cleanup_system_call
0:
	clc	8(8,%r12),BASED(cleanup_table_sysc_return)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_sysc_return+8)
	jl	cleanup_sysc_return
0:
	clc	8(8,%r12),BASED(cleanup_table_sysc_leave)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_sysc_leave+8)
	jl	cleanup_sysc_leave
0:
	clc	8(8,%r12),BASED(cleanup_table_sysc_work_loop)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_sysc_work_loop+8)
	jl	cleanup_sysc_return
0:
	clc	8(8,%r12),BASED(cleanup_table_io_return)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_io_return+8)
	jl	cleanup_io_return
0:
	clc	8(8,%r12),BASED(cleanup_table_io_leave)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_io_leave+8)
	jl	cleanup_io_leave
0:
	clc	8(8,%r12),BASED(cleanup_table_io_work_loop)
	jl	0f
	clc	8(8,%r12),BASED(cleanup_table_io_work_loop+8)
	jl	cleanup_io_return
0:
	br	%r14

cleanup_system_call:
	mvc	__LC_RETURN_PSW(16),0(%r12)
	cghi	%r12,__LC_MCK_OLD_PSW
	je	0f
	la	%r12,__LC_SAVE_AREA+32
	j	1f
0:	la	%r12,__LC_SAVE_AREA+64
1:
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	clc	__LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+8)
	jh	0f
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
0:	clc	__LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+16)
	jhe	cleanup_vtime
#endif
	clc	__LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn)
	jh	0f
	mvc	__LC_SAVE_AREA(32),0(%r12)
0:	stg	%r13,8(%r12)
	stg	%r12,__LC_SAVE_AREA+96	# argh
	SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
	lg	%r12,__LC_SAVE_AREA+96	# argh
	stg	%r15,24(%r12)
	llgh	%r7,__LC_SVC_INT_CODE
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
cleanup_vtime:
	clc	__LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+24)
	jhe	cleanup_stime
	tm	SP_PSW+1(%r15),0x01	# interrupting from user ?
	jz	cleanup_novtime
	UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
cleanup_stime:
	clc	__LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+32)
	jh	cleanup_update
	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
cleanup_update:
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
cleanup_novtime:
#endif
	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_system_call+8)
	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_system_call_insn:
	.quad	sysc_saveall
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.quad	system_call
	.quad	sysc_vtime
	.quad	sysc_stime
	.quad	sysc_update
#endif

cleanup_sysc_return:
	mvc	__LC_RETURN_PSW(8),0(%r12)
	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_sysc_return)
	la	%r12,__LC_RETURN_PSW
	br	%r14

cleanup_sysc_leave:
	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn)
	je	2f
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn+8)
	je	2f
#endif
	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)
	cghi	%r12,__LC_MCK_OLD_PSW
	jne	0f
	mvc	__LC_SAVE_AREA+64(32),SP_R12(%r15)
	j	1f
0:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)
1:	lmg	%r0,%r11,SP_R0(%r15)
	lg	%r15,SP_R15(%r15)
2:	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_sysc_leave_insn:
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.quad	sysc_leave + 16
#endif
	.quad	sysc_leave + 12

cleanup_io_return:
	mvc	__LC_RETURN_PSW(8),0(%r12)
	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_io_work_loop)
	la	%r12,__LC_RETURN_PSW
	br	%r14

cleanup_io_leave:
	clc	8(8,%r12),BASED(cleanup_io_leave_insn)
	je	2f
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
	clc	8(8,%r12),BASED(cleanup_io_leave_insn+8)
	je	2f
#endif
	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)
	cghi	%r12,__LC_MCK_OLD_PSW
	jne	0f
	mvc	__LC_SAVE_AREA+64(32),SP_R12(%r15)
	j	1f
0:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)
1:	lmg	%r0,%r11,SP_R0(%r15)
	lg	%r15,SP_R15(%r15)
2:	la	%r12,__LC_RETURN_PSW
	br	%r14
cleanup_io_leave_insn:
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	.quad	io_leave + 20
#endif
	.quad	io_leave + 16

/*
 * Integer constants
 */
		.align	4
.Lconst:
.Lc_pactive:	.long	PREEMPT_ACTIVE
.Lnr_syscalls:	.long	NR_syscalls
.L0x0130:	.short	0x130
.L0x0140:	.short	0x140
.L0x0150:	.short	0x150
.L0x0160:	.short	0x160
.L0x0170:	.short	0x170
.Lcritical_start:
		.quad	__critical_start
.Lcritical_end:
		.quad	__critical_end

		.section .rodata, "a"
#define SYSCALL(esa,esame,emu)	.long esame
sys_call_table:
#include "syscalls.S"
#undef SYSCALL

#ifdef CONFIG_COMPAT

#define SYSCALL(esa,esame,emu)	.long emu
sys_call_table_emu:
#include "syscalls.S"
#undef SYSCALL
#endif