Your IP : 172.28.240.42
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build ppc64 || ppc64le
#include "go_asm.h"
#include "textflag.h"
TEXT ·IndexByte<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-40
// R3 = byte array pointer
// R4 = length
MOVD R6, R5 // R5 = byte
BR indexbytebody<>(SB)
TEXT ·IndexByteString<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
// R3 = string
// R4 = length
// R5 = byte
BR indexbytebody<>(SB)
#ifndef GOPPC64_power9
#ifdef GOARCH_ppc64le
DATA indexbytevbperm<>+0(SB)/8, $0x3830282018100800
DATA indexbytevbperm<>+8(SB)/8, $0x7870686058504840
#else
DATA indexbytevbperm<>+0(SB)/8, $0x0008101820283038
DATA indexbytevbperm<>+8(SB)/8, $0x4048505860687078
#endif
GLOBL indexbytevbperm<>+0(SB), RODATA, $16
#endif
// Some operations are endian specific, choose the correct opcode base on GOARCH.
// Note, _VCZBEBB is only available on power9 and newer.
#ifdef GOARCH_ppc64le
#define _LDBEX MOVDBR
#define _LWBEX MOVWBR
#define _LHBEX MOVHBR
#define _VCZBEBB VCTZLSBB
#else
#define _LDBEX MOVD
#define _LWBEX MOVW
#define _LHBEX MOVH
#define _VCZBEBB VCLZLSBB
#endif
// R3 = addr of string
// R4 = len of string
// R5 = byte to find
// On exit:
// R3 = return value
TEXT indexbytebody<>(SB),NOSPLIT|NOFRAME,$0-0
CMPU R4,$32
#ifndef GOPPC64_power9
// Load VBPERMQ constant to reduce compare into an ordered bit mask.
MOVD $indexbytevbperm<>+00(SB),R16
LXVD2X (R16),V0 // Set up swap string
#endif
MTVRD R5,V1
VSPLTB $7,V1,V1 // Replicate byte across V1
BLT cmp16 // Jump to the small string case if it's <32 bytes.
CMP R4,$64,CR1
MOVD $16,R11
MOVD R3,R8
BLT CR1,cmp32 // Special case for length 32 - 63
MOVD $32,R12
MOVD $48,R6
RLDICR $0,R4,$63-6,R9 // R9 = len &^ 63
ADD R3,R9,R9 // R9 = &s[len &^ 63]
ANDCC $63,R4 // (len &= 63) cmp 0.
PCALIGN $16
loop64:
LXVD2X (R0)(R8),V2 // Scan 64 bytes at a time, starting at &s[0]
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0 // Match found at R8, jump out
LXVD2X (R11)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat1 // Match found at R8+16 bytes, jump out
LXVD2X (R12)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat2 // Match found at R8+32 bytes, jump out
LXVD2X (R6)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat3 // Match found at R8+48 bytes, jump out
ADD $64,R8
CMPU R8,R9,CR1
BNE CR1,loop64 // R8 != &s[len &^ 63]?
PCALIGN $32
BEQ notfound // Is tail length 0? CR0 is set before entering loop64.
CMP R4,$32 // Tail length >= 32, use cmp32 path.
CMP R4,$16,CR1
BGE cmp32
ADD R8,R4,R9
ADD $-16,R9
BLE CR1,cmp64_tail_gt0
cmp64_tail_gt16: // Tail length 17 - 32
LXVD2X (R0)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0
cmp64_tail_gt0: // Tail length 1 - 16
MOVD R9,R8
LXVD2X (R0)(R9),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0
BR notfound
cmp32: // Length 32 - 63
// Bytes 0 - 15
LXVD2X (R0)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0
// Bytes 16 - 31
LXVD2X (R8)(R11),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat1 // Match found at R8+16 bytes, jump out
BEQ notfound // Is length <= 32? (CR0 holds this comparison on entry to cmp32)
CMP R4,$48
ADD R4,R8,R9 // Compute &s[len(s)-16]
ADD $32,R8,R8
ADD $-16,R9,R9
ISEL CR0GT,R8,R9,R8 // R8 = len(s) <= 48 ? R9 : R8
// Bytes 33 - 47
LXVD2X (R0)(R8),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0 // match found at R8+32 bytes, jump out
BLE notfound
// Bytes 48 - 63
MOVD R9,R8 // R9 holds the final check.
LXVD2X (R0)(R9),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0 // Match found at R8+48 bytes, jump out
BR notfound
// If ISA 3.0 instructions are unavailable, we need to account for the extra 16 added by CNTLZW.
#ifndef GOPPC64_power9
#define ADJUST_FOR_CNTLZW -16
#else
#define ADJUST_FOR_CNTLZW 0
#endif
// Now, find the index of the 16B vector the match was discovered in. If CNTLZW is used
// to determine the offset into the 16B vector, it will overcount by 16. Account for it here.
foundat3:
SUB R3,R8,R3
ADD $48+ADJUST_FOR_CNTLZW,R3
BR vfound
foundat2:
SUB R3,R8,R3
ADD $32+ADJUST_FOR_CNTLZW,R3
BR vfound
foundat1:
SUB R3,R8,R3
ADD $16+ADJUST_FOR_CNTLZW,R3
BR vfound
foundat0:
SUB R3,R8,R3
ADD $0+ADJUST_FOR_CNTLZW,R3
vfound:
// Map equal values into a 16 bit value with earlier matches setting higher bits.
#ifndef GOPPC64_power9
VBPERMQ V6,V0,V6
MFVRD V6,R4
CNTLZW R4,R4
#else
#ifdef GOARCH_ppc64le
// Put the value back into LE ordering by swapping doublewords.
XXPERMDI V6,V6,$2,V6
#endif
_VCZBEBB V6,R4
#endif
ADD R3,R4,R3
RET
cmp16: // Length 16 - 31
CMPU R4,$16
ADD R4,R3,R9
BLT cmp8
ADD $-16,R9,R9 // &s[len(s)-16]
// Bytes 0 - 15
LXVD2X (R0)(R3),V2
VCMPEQUBCC V2,V1,V6
MOVD R3,R8
BNE CR6,foundat0 // Match found at R8+32 bytes, jump out
BEQ notfound
// Bytes 16 - 30
MOVD R9,R8 // R9 holds the final check.
LXVD2X (R0)(R9),V2
VCMPEQUBCC V2,V1,V6
BNE CR6,foundat0 // Match found at R8+48 bytes, jump out
BR notfound
cmp8: // Length 8 - 15
#ifdef GOPPC64_power10
// Load all the bytes into a single VSR in BE order.
SLD $56,R4,R5
LXVLL R3,R5,V2
// Compare and count the number which don't match.
VCMPEQUB V2,V1,V6
VCLZLSBB V6,R3
// If count is the number of bytes, or more. No matches are found.
CMPU R3,R4
MOVD $-1,R5
// Otherwise, the count is the index of the first match.
ISEL CR0LT,R3,R5,R3
RET
#else
RLDIMI $8,R5,$48,R5 // Replicating the byte across the register.
RLDIMI $16,R5,$32,R5
RLDIMI $32,R5,$0,R5
CMPU R4,$8
BLT cmp4
MOVD $-8,R11
ADD $-8,R4,R4
_LDBEX (R0)(R3),R10
_LDBEX (R11)(R9),R11
CMPB R10,R5,R10
CMPB R11,R5,R11
CMPU R10,$0
CMPU R11,$0,CR1
CNTLZD R10,R10
CNTLZD R11,R11
SRD $3,R10,R3
SRD $3,R11,R11
BNE found
ADD R4,R11,R4
MOVD $-1,R3
ISEL CR1EQ,R3,R4,R3
RET
cmp4: // Length 4 - 7
CMPU R4,$4
BLT cmp2
MOVD $-4,R11
ADD $-4,R4,R4
_LWBEX (R0)(R3),R10
_LWBEX (R11)(R9),R11
CMPB R10,R5,R10
CMPB R11,R5,R11
CNTLZW R10,R10
CNTLZW R11,R11
CMPU R10,$32
CMPU R11,$32,CR1
SRD $3,R10,R3
SRD $3,R11,R11
BNE found
ADD R4,R11,R4
MOVD $-1,R3
ISEL CR1EQ,R3,R4,R3
RET
cmp2: // Length 2 - 3
CMPU R4,$2
BLT cmp1
_LHBEX (R0)(R3),R10
CMPB R10,R5,R10
SLDCC $48,R10,R10
CNTLZD R10,R10
SRD $3,R10,R3
BNE found
cmp1: // Length 1
MOVD $-1,R3
ANDCC $1,R4,R31
BEQ found
MOVBZ -1(R9),R10
CMPB R10,R5,R10
ANDCC $1,R10
ADD $-1,R4
ISEL CR0EQ,R3,R4,R3
found:
RET
#endif
notfound:
MOVD $-1,R3
RET