536e062a56
Reviewed-by: mdoerr, kvn, adinn
581 lines
20 KiB
C++
581 lines
20 KiB
C++
/*
|
|
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
|
|
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
*
|
|
* This code is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 only, as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
* version 2 for more details (a copy is included in the LICENSE file that
|
|
* accompanied this code).
|
|
*
|
|
* You should have received a copy of the GNU General Public License version
|
|
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*
|
|
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
* or visit www.oracle.com if you need additional information or have any
|
|
* questions.
|
|
*
|
|
*/
|
|
|
|
#include "precompiled.hpp"
|
|
#include "asm/assembler.hpp"
|
|
#include "asm/assembler.inline.hpp"
|
|
#include "opto/c2_MacroAssembler.hpp"
|
|
#include "opto/intrinsicnode.hpp"
|
|
|
|
#ifdef PRODUCT
|
|
#define BLOCK_COMMENT(str) // nothing
|
|
#else
|
|
#define BLOCK_COMMENT(str) block_comment(str)
|
|
#endif
|
|
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
|
|
|
|
// Intrinsics for CompactStrings
|
|
|
|
// Compress char[] to byte[] by compressing 16 bytes at once.
|
|
void C2_MacroAssembler::string_compress_16(Register src, Register dst, Register cnt,
|
|
Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5,
|
|
Label& Lfailure) {
|
|
|
|
const Register tmp0 = R0;
|
|
assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5);
|
|
Label Lloop, Lslow;
|
|
|
|
// Check if cnt >= 8 (= 16 bytes)
|
|
lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF00FF00FF
|
|
srwi_(tmp2, cnt, 3);
|
|
beq(CCR0, Lslow);
|
|
ori(tmp1, tmp1, 0xFF);
|
|
rldimi(tmp1, tmp1, 32, 0);
|
|
mtctr(tmp2);
|
|
|
|
// 2x unrolled loop
|
|
bind(Lloop);
|
|
ld(tmp2, 0, src); // _0_1_2_3 (Big Endian)
|
|
ld(tmp4, 8, src); // _4_5_6_7
|
|
|
|
orr(tmp0, tmp2, tmp4);
|
|
rldicl(tmp3, tmp2, 6*8, 64-24); // _____1_2
|
|
rldimi(tmp2, tmp2, 2*8, 2*8); // _0_2_3_3
|
|
rldicl(tmp5, tmp4, 6*8, 64-24); // _____5_6
|
|
rldimi(tmp4, tmp4, 2*8, 2*8); // _4_6_7_7
|
|
|
|
andc_(tmp0, tmp0, tmp1);
|
|
bne(CCR0, Lfailure); // Not latin1.
|
|
addi(src, src, 16);
|
|
|
|
rlwimi(tmp3, tmp2, 0*8, 24, 31);// _____1_3
|
|
srdi(tmp2, tmp2, 3*8); // ____0_2_
|
|
rlwimi(tmp5, tmp4, 0*8, 24, 31);// _____5_7
|
|
srdi(tmp4, tmp4, 3*8); // ____4_6_
|
|
|
|
orr(tmp2, tmp2, tmp3); // ____0123
|
|
orr(tmp4, tmp4, tmp5); // ____4567
|
|
|
|
stw(tmp2, 0, dst);
|
|
stw(tmp4, 4, dst);
|
|
addi(dst, dst, 8);
|
|
bdnz(Lloop);
|
|
|
|
bind(Lslow); // Fallback to slow version
|
|
}
|
|
|
|
// Compress char[] to byte[]. cnt must be positive int.
|
|
void C2_MacroAssembler::string_compress(Register src, Register dst, Register cnt, Register tmp, Label& Lfailure) {
|
|
Label Lloop;
|
|
mtctr(cnt);
|
|
|
|
bind(Lloop);
|
|
lhz(tmp, 0, src);
|
|
cmplwi(CCR0, tmp, 0xff);
|
|
bgt(CCR0, Lfailure); // Not latin1.
|
|
addi(src, src, 2);
|
|
stb(tmp, 0, dst);
|
|
addi(dst, dst, 1);
|
|
bdnz(Lloop);
|
|
}
|
|
|
|
// Inflate byte[] to char[] by inflating 16 bytes at once.
|
|
void C2_MacroAssembler::string_inflate_16(Register src, Register dst, Register cnt,
|
|
Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
|
|
const Register tmp0 = R0;
|
|
assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5);
|
|
Label Lloop, Lslow;
|
|
|
|
// Check if cnt >= 8
|
|
srwi_(tmp2, cnt, 3);
|
|
beq(CCR0, Lslow);
|
|
lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF
|
|
ori(tmp1, tmp1, 0xFF);
|
|
mtctr(tmp2);
|
|
|
|
// 2x unrolled loop
|
|
bind(Lloop);
|
|
lwz(tmp2, 0, src); // ____0123 (Big Endian)
|
|
lwz(tmp4, 4, src); // ____4567
|
|
addi(src, src, 8);
|
|
|
|
rldicl(tmp3, tmp2, 7*8, 64-8); // _______2
|
|
rlwimi(tmp2, tmp2, 3*8, 16, 23);// ____0113
|
|
rldicl(tmp5, tmp4, 7*8, 64-8); // _______6
|
|
rlwimi(tmp4, tmp4, 3*8, 16, 23);// ____4557
|
|
|
|
andc(tmp0, tmp2, tmp1); // ____0_1_
|
|
rlwimi(tmp2, tmp3, 2*8, 0, 23); // _____2_3
|
|
andc(tmp3, tmp4, tmp1); // ____4_5_
|
|
rlwimi(tmp4, tmp5, 2*8, 0, 23); // _____6_7
|
|
|
|
rldimi(tmp2, tmp0, 3*8, 0*8); // _0_1_2_3
|
|
rldimi(tmp4, tmp3, 3*8, 0*8); // _4_5_6_7
|
|
|
|
std(tmp2, 0, dst);
|
|
std(tmp4, 8, dst);
|
|
addi(dst, dst, 16);
|
|
bdnz(Lloop);
|
|
|
|
bind(Lslow); // Fallback to slow version
|
|
}
|
|
|
|
// Inflate byte[] to char[]. cnt must be positive int.
|
|
void C2_MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) {
|
|
Label Lloop;
|
|
mtctr(cnt);
|
|
|
|
bind(Lloop);
|
|
lbz(tmp, 0, src);
|
|
addi(src, src, 1);
|
|
sth(tmp, 0, dst);
|
|
addi(dst, dst, 2);
|
|
bdnz(Lloop);
|
|
}
|
|
|
|
void C2_MacroAssembler::string_compare(Register str1, Register str2,
|
|
Register cnt1, Register cnt2,
|
|
Register tmp1, Register result, int ae) {
|
|
const Register tmp0 = R0,
|
|
diff = tmp1;
|
|
|
|
assert_different_registers(str1, str2, cnt1, cnt2, tmp0, tmp1, result);
|
|
Label Ldone, Lslow, Lloop, Lreturn_diff;
|
|
|
|
// Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a)
|
|
// we interchange str1 and str2 in the UL case and negate the result.
|
|
// Like this, str1 is always latin1 encoded, except for the UU case.
|
|
// In addition, we need 0 (or sign which is 0) extend.
|
|
|
|
if (ae == StrIntrinsicNode::UU) {
|
|
srwi(cnt1, cnt1, 1);
|
|
} else {
|
|
clrldi(cnt1, cnt1, 32);
|
|
}
|
|
|
|
if (ae != StrIntrinsicNode::LL) {
|
|
srwi(cnt2, cnt2, 1);
|
|
} else {
|
|
clrldi(cnt2, cnt2, 32);
|
|
}
|
|
|
|
// See if the lengths are different, and calculate min in cnt1.
|
|
// Save diff in case we need it for a tie-breaker.
|
|
subf_(diff, cnt2, cnt1); // diff = cnt1 - cnt2
|
|
// if (diff > 0) { cnt1 = cnt2; }
|
|
if (VM_Version::has_isel()) {
|
|
isel(cnt1, CCR0, Assembler::greater, /*invert*/ false, cnt2);
|
|
} else {
|
|
Label Lskip;
|
|
blt(CCR0, Lskip);
|
|
mr(cnt1, cnt2);
|
|
bind(Lskip);
|
|
}
|
|
|
|
// Rename registers
|
|
Register chr1 = result;
|
|
Register chr2 = tmp0;
|
|
|
|
// Compare multiple characters in fast loop (only implemented for same encoding).
|
|
int stride1 = 8, stride2 = 8;
|
|
if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
|
|
int log2_chars_per_iter = (ae == StrIntrinsicNode::LL) ? 3 : 2;
|
|
Label Lfastloop, Lskipfast;
|
|
|
|
srwi_(tmp0, cnt1, log2_chars_per_iter);
|
|
beq(CCR0, Lskipfast);
|
|
rldicl(cnt2, cnt1, 0, 64 - log2_chars_per_iter); // Remaining characters.
|
|
li(cnt1, 1 << log2_chars_per_iter); // Initialize for failure case: Rescan characters from current iteration.
|
|
mtctr(tmp0);
|
|
|
|
bind(Lfastloop);
|
|
ld(chr1, 0, str1);
|
|
ld(chr2, 0, str2);
|
|
cmpd(CCR0, chr1, chr2);
|
|
bne(CCR0, Lslow);
|
|
addi(str1, str1, stride1);
|
|
addi(str2, str2, stride2);
|
|
bdnz(Lfastloop);
|
|
mr(cnt1, cnt2); // Remaining characters.
|
|
bind(Lskipfast);
|
|
}
|
|
|
|
// Loop which searches the first difference character by character.
|
|
cmpwi(CCR0, cnt1, 0);
|
|
beq(CCR0, Lreturn_diff);
|
|
bind(Lslow);
|
|
mtctr(cnt1);
|
|
|
|
switch (ae) {
|
|
case StrIntrinsicNode::LL: stride1 = 1; stride2 = 1; break;
|
|
case StrIntrinsicNode::UL: // fallthru (see comment above)
|
|
case StrIntrinsicNode::LU: stride1 = 1; stride2 = 2; break;
|
|
case StrIntrinsicNode::UU: stride1 = 2; stride2 = 2; break;
|
|
default: ShouldNotReachHere(); break;
|
|
}
|
|
|
|
bind(Lloop);
|
|
if (stride1 == 1) { lbz(chr1, 0, str1); } else { lhz(chr1, 0, str1); }
|
|
if (stride2 == 1) { lbz(chr2, 0, str2); } else { lhz(chr2, 0, str2); }
|
|
subf_(result, chr2, chr1); // result = chr1 - chr2
|
|
bne(CCR0, Ldone);
|
|
addi(str1, str1, stride1);
|
|
addi(str2, str2, stride2);
|
|
bdnz(Lloop);
|
|
|
|
// If strings are equal up to min length, return the length difference.
|
|
bind(Lreturn_diff);
|
|
mr(result, diff);
|
|
|
|
// Otherwise, return the difference between the first mismatched chars.
|
|
bind(Ldone);
|
|
if (ae == StrIntrinsicNode::UL) {
|
|
neg(result, result); // Negate result (see note above).
|
|
}
|
|
}
|
|
|
|
void C2_MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2,
|
|
Register limit, Register tmp1, Register result, bool is_byte) {
|
|
const Register tmp0 = R0;
|
|
assert_different_registers(ary1, ary2, limit, tmp0, tmp1, result);
|
|
Label Ldone, Lskiploop, Lloop, Lfastloop, Lskipfast;
|
|
bool limit_needs_shift = false;
|
|
|
|
if (is_array_equ) {
|
|
const int length_offset = arrayOopDesc::length_offset_in_bytes();
|
|
const int base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR);
|
|
|
|
// Return true if the same array.
|
|
cmpd(CCR0, ary1, ary2);
|
|
beq(CCR0, Lskiploop);
|
|
|
|
// Return false if one of them is NULL.
|
|
cmpdi(CCR0, ary1, 0);
|
|
cmpdi(CCR1, ary2, 0);
|
|
li(result, 0);
|
|
cror(CCR0, Assembler::equal, CCR1, Assembler::equal);
|
|
beq(CCR0, Ldone);
|
|
|
|
// Load the lengths of arrays.
|
|
lwz(limit, length_offset, ary1);
|
|
lwz(tmp0, length_offset, ary2);
|
|
|
|
// Return false if the two arrays are not equal length.
|
|
cmpw(CCR0, limit, tmp0);
|
|
bne(CCR0, Ldone);
|
|
|
|
// Load array addresses.
|
|
addi(ary1, ary1, base_offset);
|
|
addi(ary2, ary2, base_offset);
|
|
} else {
|
|
limit_needs_shift = !is_byte;
|
|
li(result, 0); // Assume not equal.
|
|
}
|
|
|
|
// Rename registers
|
|
Register chr1 = tmp0;
|
|
Register chr2 = tmp1;
|
|
|
|
// Compare 8 bytes per iteration in fast loop.
|
|
const int log2_chars_per_iter = is_byte ? 3 : 2;
|
|
|
|
srwi_(tmp0, limit, log2_chars_per_iter + (limit_needs_shift ? 1 : 0));
|
|
beq(CCR0, Lskipfast);
|
|
mtctr(tmp0);
|
|
|
|
bind(Lfastloop);
|
|
ld(chr1, 0, ary1);
|
|
ld(chr2, 0, ary2);
|
|
addi(ary1, ary1, 8);
|
|
addi(ary2, ary2, 8);
|
|
cmpd(CCR0, chr1, chr2);
|
|
bne(CCR0, Ldone);
|
|
bdnz(Lfastloop);
|
|
|
|
bind(Lskipfast);
|
|
rldicl_(limit, limit, limit_needs_shift ? 64 - 1 : 0, 64 - log2_chars_per_iter); // Remaining characters.
|
|
beq(CCR0, Lskiploop);
|
|
mtctr(limit);
|
|
|
|
// Character by character.
|
|
bind(Lloop);
|
|
if (is_byte) {
|
|
lbz(chr1, 0, ary1);
|
|
lbz(chr2, 0, ary2);
|
|
addi(ary1, ary1, 1);
|
|
addi(ary2, ary2, 1);
|
|
} else {
|
|
lhz(chr1, 0, ary1);
|
|
lhz(chr2, 0, ary2);
|
|
addi(ary1, ary1, 2);
|
|
addi(ary2, ary2, 2);
|
|
}
|
|
cmpw(CCR0, chr1, chr2);
|
|
bne(CCR0, Ldone);
|
|
bdnz(Lloop);
|
|
|
|
bind(Lskiploop);
|
|
li(result, 1); // All characters are equal.
|
|
bind(Ldone);
|
|
}
|
|
|
|
void C2_MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt,
|
|
Register needle, ciTypeArray* needle_values, Register needlecnt, int needlecntval,
|
|
Register tmp1, Register tmp2, Register tmp3, Register tmp4, int ae) {
|
|
|
|
// Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite!
|
|
Label L_TooShort, L_Found, L_NotFound, L_End;
|
|
Register last_addr = haycnt, // Kill haycnt at the beginning.
|
|
addr = tmp1,
|
|
n_start = tmp2,
|
|
ch1 = tmp3,
|
|
ch2 = R0;
|
|
|
|
assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
|
|
const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2;
|
|
const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1;
|
|
|
|
// **************************************************************************************************
|
|
// Prepare for main loop: optimized for needle count >=2, bail out otherwise.
|
|
// **************************************************************************************************
|
|
|
|
// Compute last haystack addr to use if no match gets found.
|
|
clrldi(haycnt, haycnt, 32); // Ensure positive int is valid as 64 bit value.
|
|
addi(addr, haystack, -h_csize); // Accesses use pre-increment.
|
|
if (needlecntval == 0) { // variable needlecnt
|
|
cmpwi(CCR6, needlecnt, 2);
|
|
clrldi(needlecnt, needlecnt, 32); // Ensure positive int is valid as 64 bit value.
|
|
blt(CCR6, L_TooShort); // Variable needlecnt: handle short needle separately.
|
|
}
|
|
|
|
if (n_csize == 2) { lwz(n_start, 0, needle); } else { lhz(n_start, 0, needle); } // Load first 2 characters of needle.
|
|
|
|
if (needlecntval == 0) { // variable needlecnt
|
|
subf(ch1, needlecnt, haycnt); // Last character index to compare is haycnt-needlecnt.
|
|
addi(needlecnt, needlecnt, -2); // Rest of needle.
|
|
} else { // constant needlecnt
|
|
guarantee(needlecntval != 1, "IndexOf with single-character needle must be handled separately");
|
|
assert((needlecntval & 0x7fff) == needlecntval, "wrong immediate");
|
|
addi(ch1, haycnt, -needlecntval); // Last character index to compare is haycnt-needlecnt.
|
|
if (needlecntval > 3) { li(needlecnt, needlecntval - 2); } // Rest of needle.
|
|
}
|
|
|
|
if (h_csize == 2) { slwi(ch1, ch1, 1); } // Scale to number of bytes.
|
|
|
|
if (ae ==StrIntrinsicNode::UL) {
|
|
srwi(tmp4, n_start, 1*8); // ___0
|
|
rlwimi(n_start, tmp4, 2*8, 0, 23); // _0_1
|
|
}
|
|
|
|
add(last_addr, haystack, ch1); // Point to last address to compare (haystack+2*(haycnt-needlecnt)).
|
|
|
|
// Main Loop (now we have at least 2 characters).
|
|
Label L_OuterLoop, L_InnerLoop, L_FinalCheck, L_Comp1, L_Comp2;
|
|
bind(L_OuterLoop); // Search for 1st 2 characters.
|
|
Register addr_diff = tmp4;
|
|
subf(addr_diff, addr, last_addr); // Difference between already checked address and last address to check.
|
|
addi(addr, addr, h_csize); // This is the new address we want to use for comparing.
|
|
srdi_(ch2, addr_diff, h_csize);
|
|
beq(CCR0, L_FinalCheck); // 2 characters left?
|
|
mtctr(ch2); // num of characters / 2
|
|
bind(L_InnerLoop); // Main work horse (2x unrolled search loop)
|
|
if (h_csize == 2) { // Load 2 characters of haystack (ignore alignment).
|
|
lwz(ch1, 0, addr);
|
|
lwz(ch2, 2, addr);
|
|
} else {
|
|
lhz(ch1, 0, addr);
|
|
lhz(ch2, 1, addr);
|
|
}
|
|
cmpw(CCR0, ch1, n_start); // Compare 2 characters (1 would be sufficient but try to reduce branches to CompLoop).
|
|
cmpw(CCR1, ch2, n_start);
|
|
beq(CCR0, L_Comp1); // Did we find the needle start?
|
|
beq(CCR1, L_Comp2);
|
|
addi(addr, addr, 2 * h_csize);
|
|
bdnz(L_InnerLoop);
|
|
bind(L_FinalCheck);
|
|
andi_(addr_diff, addr_diff, h_csize); // Remaining characters not covered by InnerLoop: (num of characters) & 1.
|
|
beq(CCR0, L_NotFound);
|
|
if (h_csize == 2) { lwz(ch1, 0, addr); } else { lhz(ch1, 0, addr); } // One position left at which we have to compare.
|
|
cmpw(CCR1, ch1, n_start);
|
|
beq(CCR1, L_Comp1);
|
|
bind(L_NotFound);
|
|
li(result, -1); // not found
|
|
b(L_End);
|
|
|
|
// **************************************************************************************************
|
|
// Special Case: unfortunately, the variable needle case can be called with needlecnt<2
|
|
// **************************************************************************************************
|
|
if (needlecntval == 0) { // We have to handle these cases separately.
|
|
Label L_OneCharLoop;
|
|
bind(L_TooShort);
|
|
mtctr(haycnt);
|
|
if (n_csize == 2) { lhz(n_start, 0, needle); } else { lbz(n_start, 0, needle); } // First character of needle
|
|
bind(L_OneCharLoop);
|
|
if (h_csize == 2) { lhzu(ch1, 2, addr); } else { lbzu(ch1, 1, addr); }
|
|
cmpw(CCR1, ch1, n_start);
|
|
beq(CCR1, L_Found); // Did we find the one character needle?
|
|
bdnz(L_OneCharLoop);
|
|
li(result, -1); // Not found.
|
|
b(L_End);
|
|
}
|
|
|
|
// **************************************************************************************************
|
|
// Regular Case Part II: compare rest of needle (first 2 characters have been compared already)
|
|
// **************************************************************************************************
|
|
|
|
// Compare the rest
|
|
bind(L_Comp2);
|
|
addi(addr, addr, h_csize); // First comparison has failed, 2nd one hit.
|
|
bind(L_Comp1); // Addr points to possible needle start.
|
|
if (needlecntval != 2) { // Const needlecnt==2?
|
|
if (needlecntval != 3) {
|
|
if (needlecntval == 0) { beq(CCR6, L_Found); } // Variable needlecnt==2?
|
|
Register n_ind = tmp4,
|
|
h_ind = n_ind;
|
|
li(n_ind, 2 * n_csize); // First 2 characters are already compared, use index 2.
|
|
mtctr(needlecnt); // Decremented by 2, still > 0.
|
|
Label L_CompLoop;
|
|
bind(L_CompLoop);
|
|
if (ae ==StrIntrinsicNode::UL) {
|
|
h_ind = ch1;
|
|
sldi(h_ind, n_ind, 1);
|
|
}
|
|
if (n_csize == 2) { lhzx(ch2, needle, n_ind); } else { lbzx(ch2, needle, n_ind); }
|
|
if (h_csize == 2) { lhzx(ch1, addr, h_ind); } else { lbzx(ch1, addr, h_ind); }
|
|
cmpw(CCR1, ch1, ch2);
|
|
bne(CCR1, L_OuterLoop);
|
|
addi(n_ind, n_ind, n_csize);
|
|
bdnz(L_CompLoop);
|
|
} else { // No loop required if there's only one needle character left.
|
|
if (n_csize == 2) { lhz(ch2, 2 * 2, needle); } else { lbz(ch2, 2 * 1, needle); }
|
|
if (h_csize == 2) { lhz(ch1, 2 * 2, addr); } else { lbz(ch1, 2 * 1, addr); }
|
|
cmpw(CCR1, ch1, ch2);
|
|
bne(CCR1, L_OuterLoop);
|
|
}
|
|
}
|
|
// Return index ...
|
|
bind(L_Found);
|
|
subf(result, haystack, addr); // relative to haystack, ...
|
|
if (h_csize == 2) { srdi(result, result, 1); } // in characters.
|
|
bind(L_End);
|
|
} // string_indexof
|
|
|
|
void C2_MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt,
|
|
Register needle, jchar needleChar, Register tmp1, Register tmp2, bool is_byte) {
|
|
assert_different_registers(haystack, haycnt, needle, tmp1, tmp2);
|
|
|
|
Label L_InnerLoop, L_FinalCheck, L_Found1, L_Found2, L_NotFound, L_End;
|
|
Register addr = tmp1,
|
|
ch1 = tmp2,
|
|
ch2 = R0;
|
|
|
|
const int h_csize = is_byte ? 1 : 2;
|
|
|
|
//4:
|
|
srwi_(tmp2, haycnt, 1); // Shift right by exact_log2(UNROLL_FACTOR).
|
|
mr(addr, haystack);
|
|
beq(CCR0, L_FinalCheck);
|
|
mtctr(tmp2); // Move to count register.
|
|
//8:
|
|
bind(L_InnerLoop); // Main work horse (2x unrolled search loop).
|
|
if (!is_byte) {
|
|
lhz(ch1, 0, addr);
|
|
lhz(ch2, 2, addr);
|
|
} else {
|
|
lbz(ch1, 0, addr);
|
|
lbz(ch2, 1, addr);
|
|
}
|
|
(needle != R0) ? cmpw(CCR0, ch1, needle) : cmplwi(CCR0, ch1, (unsigned int)needleChar);
|
|
(needle != R0) ? cmpw(CCR1, ch2, needle) : cmplwi(CCR1, ch2, (unsigned int)needleChar);
|
|
beq(CCR0, L_Found1); // Did we find the needle?
|
|
beq(CCR1, L_Found2);
|
|
addi(addr, addr, 2 * h_csize);
|
|
bdnz(L_InnerLoop);
|
|
//16:
|
|
bind(L_FinalCheck);
|
|
andi_(R0, haycnt, 1);
|
|
beq(CCR0, L_NotFound);
|
|
if (!is_byte) { lhz(ch1, 0, addr); } else { lbz(ch1, 0, addr); } // One position left at which we have to compare.
|
|
(needle != R0) ? cmpw(CCR1, ch1, needle) : cmplwi(CCR1, ch1, (unsigned int)needleChar);
|
|
beq(CCR1, L_Found1);
|
|
//21:
|
|
bind(L_NotFound);
|
|
li(result, -1); // Not found.
|
|
b(L_End);
|
|
|
|
bind(L_Found2);
|
|
addi(addr, addr, h_csize);
|
|
//24:
|
|
bind(L_Found1); // Return index ...
|
|
subf(result, haystack, addr); // relative to haystack, ...
|
|
if (!is_byte) { srdi(result, result, 1); } // in characters.
|
|
bind(L_End);
|
|
} // string_indexof_char
|
|
|
|
|
|
void C2_MacroAssembler::has_negatives(Register src, Register cnt, Register result,
|
|
Register tmp1, Register tmp2) {
|
|
const Register tmp0 = R0;
|
|
assert_different_registers(src, result, cnt, tmp0, tmp1, tmp2);
|
|
Label Lfastloop, Lslow, Lloop, Lnoneg, Ldone;
|
|
|
|
// Check if cnt >= 8 (= 16 bytes)
|
|
lis(tmp1, (int)(short)0x8080); // tmp1 = 0x8080808080808080
|
|
srwi_(tmp2, cnt, 4);
|
|
li(result, 1); // Assume there's a negative byte.
|
|
beq(CCR0, Lslow);
|
|
ori(tmp1, tmp1, 0x8080);
|
|
rldimi(tmp1, tmp1, 32, 0);
|
|
mtctr(tmp2);
|
|
|
|
// 2x unrolled loop
|
|
bind(Lfastloop);
|
|
ld(tmp2, 0, src);
|
|
ld(tmp0, 8, src);
|
|
|
|
orr(tmp0, tmp2, tmp0);
|
|
|
|
and_(tmp0, tmp0, tmp1);
|
|
bne(CCR0, Ldone); // Found negative byte.
|
|
addi(src, src, 16);
|
|
|
|
bdnz(Lfastloop);
|
|
|
|
bind(Lslow); // Fallback to slow version
|
|
rldicl_(tmp0, cnt, 0, 64-4);
|
|
beq(CCR0, Lnoneg);
|
|
mtctr(tmp0);
|
|
bind(Lloop);
|
|
lbz(tmp0, 0, src);
|
|
addi(src, src, 1);
|
|
andi_(tmp0, tmp0, 0x80);
|
|
bne(CCR0, Ldone); // Found negative byte.
|
|
bdnz(Lloop);
|
|
bind(Lnoneg);
|
|
li(result, 0);
|
|
|
|
bind(Ldone);
|
|
}
|
|
|