diff --git a/src/hotspot/cpu/x86/peephole_x86_64.cpp b/src/hotspot/cpu/x86/peephole_x86_64.cpp new file mode 100644 index 00000000000..ab58c566ea4 --- /dev/null +++ b/src/hotspot/cpu/x86/peephole_x86_64.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#ifdef COMPILER2 + +#include "peephole_x86_64.hpp" + +// This function transforms the shapes +// mov d, s1; add d, s2 into +// lea d, [s1 + s2] and +// mov d, s1; shl d, s2 into +// lea d, [s1 << s2] with s2 = 1, 2, 3 +bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_, + MachNode* (*new_root)(), uint inst0_rule, bool imm) { + MachNode* inst0 = block->get_node(block_index)->as_Mach(); + assert(inst0->rule() == inst0_rule, "sanity"); + + OptoReg::Name dst = ra_->get_reg_first(inst0); + MachNode* inst1 = nullptr; + OptoReg::Name src1 = OptoReg::Bad; + + if (inst0->in(1)->is_MachSpillCopy()) { + OptoReg::Name in = ra_->get_reg_first(inst0->in(1)->in(1)); + if (OptoReg::is_reg(in) && OptoReg::as_VMReg(in)->is_Register()) { + inst1 = inst0->in(1)->as_Mach(); + src1 = in; + } + } + if (inst1 == nullptr) { + return false; + } + assert(dst != src1, ""); + + // Only coalesce if inst1 is immediately followed by inst0 + // Can be improved for more general cases + if (block_index < 1 || block->get_node(block_index - 1) != inst1) { + return false; + } + int inst1_index = block_index - 1; + Node* inst2; + if (imm) { + inst2 = nullptr; + } else { + inst2 = inst0->in(2); + if (inst2 == inst1) { + inst2 = inst2->in(1); + } + } + + // See VM_Version::supports_fast_3op_lea() + if (!imm) { + Register rsrc1 = OptoReg::as_VMReg(src1)->as_Register(); + Register rsrc2 = OptoReg::as_VMReg(ra_->get_reg_first(inst2))->as_Register(); + if ((rsrc1 == rbp || rsrc1 == r13) && (rsrc2 == rbp || rsrc2 == r13)) { + return false; + } + } + + // Go down the block to find the output proj node (the flag output) of inst0 + int proj_index = -1; + Node* proj = nullptr; + for (uint pos = block_index + 1; pos < block->number_of_nodes(); pos++) { + Node* curr = block->get_node(pos); + if (curr->is_MachProj() && curr->in(0) == inst0) { + proj_index = pos; + proj = curr; + break; + } + } + assert(proj != nullptr, ""); + // If some node uses the flag, cannot remove + if (proj->outcnt() > 0) { + return false; + } + + MachNode* root = new_root(); + // Assign register for the newly allocated node + ra_->set_oop(root, ra_->is_oop(inst0)); + ra_->set_pair(root->_idx, ra_->get_reg_second(inst0), ra_->get_reg_first(inst0)); + + // Set input and output for the node + root->add_req(inst0->in(0)); + root->add_req(inst1->in(1)); + // No input for constant after matching + if (!imm) { + root->add_req(inst2); + } + inst0->replace_by(root); + proj->set_req(0, inst0); + + // Initialize the operand array + root->_opnds[0] = inst0->_opnds[0]->clone(); + root->_opnds[1] = inst0->_opnds[1]->clone(); + root->_opnds[2] = inst0->_opnds[2]->clone(); + + // Modify the block + inst0->set_removed(); + inst1->set_removed(); + block->remove_node(proj_index); + block->remove_node(block_index); + block->remove_node(inst1_index); + block->insert_node(root, block_index - 1); + + // Modify the CFG + cfg_->map_node_to_block(inst0, nullptr); + cfg_->map_node_to_block(inst1, nullptr); + cfg_->map_node_to_block(proj, nullptr); + cfg_->map_node_to_block(root, block); + + return true; +} + +bool Peephole::lea_coalesce_reg(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_, + MachNode* (*new_root)(), uint inst0_rule) { + return lea_coalesce_helper(block, block_index, cfg_, ra_, new_root, inst0_rule, false); +} + +bool Peephole::lea_coalesce_imm(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_, + MachNode* (*new_root)(), uint inst0_rule) { + return lea_coalesce_helper(block, block_index, cfg_, ra_, new_root, inst0_rule, true); +} + +#endif // COMPILER2 diff --git a/src/hotspot/cpu/x86/peephole_x86_64.hpp b/src/hotspot/cpu/x86/peephole_x86_64.hpp new file mode 100644 index 00000000000..65227820173 --- /dev/null +++ b/src/hotspot/cpu/x86/peephole_x86_64.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_PEEPHOLE_X86_64_HPP +#define CPU_X86_PEEPHOLE_X86_64_HPP + +#include "opto/machnode.hpp" +#include "opto/regalloc.hpp" + +class Peephole { +public: + static bool lea_coalesce_reg(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_, + MachNode* (*new_root)(), uint inst0_rule); + static bool lea_coalesce_imm(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_, + MachNode* (*new_root)(), uint inst0_rule); +}; + +#endif // CPU_X86_PEEPHOLE_X86_64_HPP diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad index 57ffda3df49..017ce230ba4 100644 --- a/src/hotspot/cpu/x86/x86_32.ad +++ b/src/hotspot/cpu/x86/x86_32.ad @@ -657,8 +657,8 @@ void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const { } st->print_cr("POPL EBP"); st->print("\t"); if (do_polling() && C->is_method_compilation()) { - st->print("CMPL rsp, poll_offset[thread] \n\t" - "JA #safepoint_stub\t" + st->print("CMPL rsp, poll_offset[thread] \n\t" + "JA #safepoint_stub\t" "# Safepoint: poll for GC"); } } diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 04e13271931..995ed35de0c 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -319,6 +319,14 @@ reg_class int_rdi_reg(RDI); //----------SOURCE BLOCK------------------------------------------------------- // This is a block of C++ code which provides values, functions, and // definitions necessary in the rest of the architecture description + +source_hpp %{ + +#include "peephole_x86_64.hpp" + +%} + +// Register masks source_hpp %{ extern RegMask _ANY_REG_mask; @@ -961,8 +969,8 @@ void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const st->print_cr("popq rbp"); if (do_polling() && C->is_method_compilation()) { st->print("\t"); - st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t" - "ja #safepoint_stub\t" + st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t" + "ja #safepoint_stub\t" "# Safepoint: poll for GC"); } } @@ -9001,6 +9009,19 @@ instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg c %} // Integer Shift Instructions +// Shift Left by one, two, three +instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr) +%{ + match(Set dst (LShiftI dst shift)); + effect(KILL cr); + + format %{ "sall $dst, $shift" %} + ins_encode %{ + __ sall($dst$$Register, $shift$$constant); + %} + ins_pipe(ialu_reg); +%} + // Shift Left by 8-bit immediate instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr) %{ @@ -9235,6 +9256,19 @@ instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift) %} // Long Shift Instructions +// Shift Left by one, two, three +instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr) +%{ + match(Set dst (LShiftL dst shift)); + effect(KILL cr); + + format %{ "salq $dst, $shift" %} + ins_encode %{ + __ salq($dst$$Register, $shift$$constant); + %} + ins_pipe(ialu_reg); +%} + // Shift Left by 8-bit immediate instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr) %{ @@ -13541,8 +13575,21 @@ instruct tlsLoadP(r15_RegP dst) %{ // These must follow all instruction definitions as they use the names // defined in the instructions definitions. // +// peeppredicate ( rule_predicate ); +// // the predicate unless which the peephole rule will be ignored +// // peepmatch ( root_instr_name [preceding_instruction]* ); // +// peepprocedure ( procedure_name ); +// // provide a procedure name to perform the optimization, the procedure should +// // reside in the architecture dependent peephole file, the method has the +// // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...) +// // with the arguments being the basic block, the current node index inside the +// // block, the register allocator, the functions upon invoked return a new node +// // defined in peepreplace, and the rules of the nodes appearing in the +// // corresponding peepmatch, the function return true if successful, else +// // return false +// // peepconstraint %{ // (instruction_number.operand_name relational_op instruction_number.operand_name // [, ...] ); @@ -13563,10 +13610,7 @@ instruct tlsLoadP(r15_RegP dst) %{ // // ---------CURRENT LIMITATIONS---------------------------------------------- // -// Only match adjacent instructions in same basic block -// Only equality constraints -// Only constraints between operands, not (0.dest_reg == RAX_enc) -// Only one replacement instruction +// Only transformations inside a basic block (do we need more for peephole) // // ---------EXAMPLE---------------------------------------------------------- // @@ -13582,8 +13626,21 @@ instruct tlsLoadP(r15_RegP dst) %{ // effect(KILL cr); // %} // +// instruct leaI_rReg_immI(rRegI dst, immI_1 src) +// %{ +// match(Set dst (AddI dst src)); +// %} +// +// 1. Simple replacement +// - Only match adjacent instructions in same basic block +// - Only equality constraints +// - Only constraints between operands, not (0.dest_reg == RAX_enc) +// - Only one replacement instruction +// // // Change (inc mov) to lea // peephole %{ +// // lea should only be emitted when beneficial +// peeppredicate( VM_Version::supports_fast_2op_lea() ); // // increment preceded by register-register move // peepmatch ( incI_rReg movI ); // // require that the destination register of the increment @@ -13594,83 +13651,199 @@ instruct tlsLoadP(r15_RegP dst) %{ // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) ); // %} // - -// Implementation no longer uses movX instructions since -// machine-independent system no longer uses CopyX nodes. +// 2. Procedural replacement +// - More flexible finding relevent nodes +// - More flexible constraints +// - More flexible transformations +// - May utilise architecture-dependent API more effectively +// - Currently only one replacement instruction due to adlc parsing capabilities // -// peephole -// %{ -// peepmatch (incI_rReg movI); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaI_rReg_immI(0.dst 1.src 0.src)); +// // Change (inc mov) to lea +// peephole %{ +// // lea should only be emitted when beneficial +// peeppredicate( VM_Version::supports_fast_2op_lea() ); +// // the rule numbers of these nodes inside are passed into the function below +// peepmatch ( incI_rReg movI ); +// // the method that takes the responsibility of transformation +// peepprocedure ( inc_mov_to_lea ); +// // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this +// // node is passed into the function above +// peepreplace ( leaI_rReg_immI() ); // %} -// peephole -// %{ -// peepmatch (decI_rReg movI); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaI_rReg_immI(0.dst 1.src 0.src)); -// %} - -// peephole -// %{ -// peepmatch (addI_rReg_imm movI); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaI_rReg_immI(0.dst 1.src 0.src)); -// %} - -// peephole -// %{ -// peepmatch (incL_rReg movL); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaL_rReg_immL(0.dst 1.src 0.src)); -// %} - -// peephole -// %{ -// peepmatch (decL_rReg movL); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaL_rReg_immL(0.dst 1.src 0.src)); -// %} - -// peephole -// %{ -// peepmatch (addL_rReg_imm movL); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaL_rReg_immL(0.dst 1.src 0.src)); -// %} - -// peephole -// %{ -// peepmatch (addP_rReg_imm movP); -// peepconstraint (0.dst == 1.dst); -// peepreplace (leaP_rReg_imm(0.dst 1.src 0.src)); -// %} - -// // Change load of spilled value to only a spill -// instruct storeI(memory mem, rRegI src) -// %{ -// match(Set mem (StoreI mem src)); -// %} -// -// instruct loadI(rRegI dst, memory mem) -// %{ -// match(Set dst (LoadI mem)); -// %} -// - -peephole +// These instructions is not matched by the matcher but used by the peephole +instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2) %{ - peepmatch (loadI storeI); - peepconstraint (1.src == 0.dst, 1.mem == 0.mem); - peepreplace (storeI(1.mem 1.mem 1.src)); + predicate(false); + match(Set dst (AddI src1 src2)); + format %{ "leal $dst, [$src1 + $src2]" %} + ins_encode %{ + Register dst = $dst$$Register; + Register src1 = $src1$$Register; + Register src2 = $src2$$Register; + if (src1 != rbp && src1 != r13) { + __ leal(dst, Address(src1, src2, Address::times_1)); + } else { + assert(src2 != rbp && src2 != r13, ""); + __ leal(dst, Address(src2, src1, Address::times_1)); + } + %} + ins_pipe(ialu_reg_reg); +%} + +instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2) +%{ + predicate(false); + match(Set dst (AddI src1 src2)); + format %{ "leal $dst, [$src1 + $src2]" %} + ins_encode %{ + __ leal($dst$$Register, Address($src1$$Register, $src2$$constant)); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift) +%{ + predicate(false); + match(Set dst (LShiftI src shift)); + format %{ "leal $dst, [$src << $shift]" %} + ins_encode %{ + Address::ScaleFactor scale = static_cast($shift$$constant); + Register src = $src$$Register; + if (scale == Address::times_2 && src != rbp && src != r13) { + __ leal($dst$$Register, Address(src, src, Address::times_1)); + } else { + __ leal($dst$$Register, Address(noreg, src, scale)); + } + %} + ins_pipe(ialu_reg_reg); +%} + +instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2) +%{ + predicate(false); + match(Set dst (AddL src1 src2)); + format %{ "leaq $dst, [$src1 + $src2]" %} + ins_encode %{ + Register dst = $dst$$Register; + Register src1 = $src1$$Register; + Register src2 = $src2$$Register; + if (src1 != rbp && src1 != r13) { + __ leaq(dst, Address(src1, src2, Address::times_1)); + } else { + assert(src2 != rbp && src2 != r13, ""); + __ leaq(dst, Address(src2, src1, Address::times_1)); + } + %} + ins_pipe(ialu_reg_reg); +%} + +instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2) +%{ + predicate(false); + match(Set dst (AddL src1 src2)); + format %{ "leaq $dst, [$src1 + $src2]" %} + ins_encode %{ + __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant)); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift) +%{ + predicate(false); + match(Set dst (LShiftL src shift)); + format %{ "leaq $dst, [$src << $shift]" %} + ins_encode %{ + Address::ScaleFactor scale = static_cast($shift$$constant); + Register src = $src$$Register; + if (scale == Address::times_2 && src != rbp && src != r13) { + __ leaq($dst$$Register, Address(src, src, Address::times_1)); + } else { + __ leaq($dst$$Register, Address(noreg, src, scale)); + } + %} + ins_pipe(ialu_reg_reg); %} peephole %{ - peepmatch (loadL storeL); - peepconstraint (1.src == 0.dst, 1.mem == 0.mem); - peepreplace (storeL(1.mem 1.mem 1.src)); + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (addI_rReg); + peepprocedure (lea_coalesce_reg); + peepreplace (leaI_rReg_rReg_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (addI_rReg_imm); + peepprocedure (lea_coalesce_imm); + peepreplace (leaI_rReg_immI_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (incI_rReg); + peepprocedure (lea_coalesce_imm); + peepreplace (leaI_rReg_immI_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (decI_rReg); + peepprocedure (lea_coalesce_imm); + peepreplace (leaI_rReg_immI_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (salI_rReg_immI2); + peepprocedure (lea_coalesce_imm); + peepreplace (leaI_rReg_immI2_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (addL_rReg); + peepprocedure (lea_coalesce_reg); + peepreplace (leaL_rReg_rReg_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (addL_rReg_imm); + peepprocedure (lea_coalesce_imm); + peepreplace (leaL_rReg_immL32_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (incL_rReg); + peepprocedure (lea_coalesce_imm); + peepreplace (leaL_rReg_immL32_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (decL_rReg); + peepprocedure (lea_coalesce_imm); + peepreplace (leaL_rReg_immL32_peep()); +%} + +peephole +%{ + peeppredicate(VM_Version::supports_fast_2op_lea()); + peepmatch (salL_rReg_immI2); + peepprocedure (lea_coalesce_imm); + peepreplace (leaL_rReg_immI2_peep()); %} //----------SMARTSPILL RULES--------------------------------------------------- diff --git a/src/hotspot/share/adlc/adlparse.cpp b/src/hotspot/share/adlc/adlparse.cpp index 26a3c3a7d5f..adf5b731589 100644 --- a/src/hotspot/share/adlc/adlparse.cpp +++ b/src/hotspot/share/adlc/adlparse.cpp @@ -2046,14 +2046,20 @@ void ADLParser::peep_parse(void) { return; } // check for legal subsections of peephole rule - if (strcmp(token,"peepmatch")==0) { + if (strcmp(token,"peeppredicate")==0) { + peep_predicate_parse(*peep); } + else if (strcmp(token,"peepmatch")==0) { peep_match_parse(*peep); } + else if (strcmp(token, "peepprocedure")==0) { + peep_procedure_parse(*peep); } else if (strcmp(token,"peepconstraint")==0) { peep_constraint_parse(*peep); } else if (strcmp(token,"peepreplace")==0) { peep_replace_parse(*peep); } else { - parse_err(SYNERR, "expected peepmatch, peepconstraint, or peepreplace for identifier %s.\n", token); + parse_err(SYNERR, + "expected peeppreddicate, peepmatch, peepprocedure, peepconstraint, peepreplace, received %s.\n", + token); } skipws(); } @@ -2537,6 +2543,31 @@ InstructForm *ADLParser::peep_match_child_parse(PeepMatch &match, int parent, in return NULL; } +//---------------------------peep-predicate-parse------------------------------ +// Syntax for a peeppredicate rule +// +// peeppredicate ( predicate ); +// +void ADLParser::peep_predicate_parse(Peephole& peep) { + + skipws(); + char* rule = nullptr; + if ( (rule = get_paren_expr("pred expression", true)) == nullptr ) { + parse_err(SYNERR, "incorrect or missing expression for 'peeppredicate'\n"); + return; + } + if (_curchar != ';') { + parse_err(SYNERR, "missing ';' in peeppredicate definition\n"); + return; + } + next_char(); // skip ';' + skipws(); + + // Construct PeepPredicate + PeepPredicate* predicate = new PeepPredicate(rule); + peep.add_predicate(predicate); +} + //------------------------------peep_match_parse------------------------------- // Syntax for a peepmatch rule // @@ -2583,6 +2614,46 @@ void ADLParser::peep_match_parse(Peephole &peep) { root->append_peephole(&peep); } +//---------------------------peep-procedure-parse------------------------------ +// Syntax for a peepprocedure rule +// +// peeppredicate ( function_name ); +// +void ADLParser::peep_procedure_parse(Peephole& peep) { + + skipws(); + // Check for open paren + if (_curchar != '(') { + parse_err(SYNERR, "missing '(' at start of peepprocedure rule.\n"); + return; + } + next_char(); // skip '(' + skipws(); + + char* name = nullptr; + if ( (name = get_ident_dup()) == nullptr ) { + parse_err(SYNERR, "incorrect or missing expression for 'peepprocedure'\n"); + return; + } + + skipws(); + if (_curchar != ')') { + parse_err(SYNERR, "peepprocedure should contain a single identifier only\n"); + return; + } + next_char(); // skip ')' + if (_curchar != ';') { + parse_err(SYNERR, "missing ';' in peepprocedure definition\n"); + return; + } + next_char(); // skip ';' + skipws(); + + // Construct PeepProcedure + PeepProcedure* procedure = new PeepProcedure(name); + peep.add_procedure(procedure); +} + //------------------------------peep_constraint_parse-------------------------- // Syntax for a peepconstraint rule // A parenthesized list of relations between operands in peepmatch subtree @@ -2629,7 +2700,7 @@ void ADLParser::peep_constraint_parse(Peephole &peep) { skipws(); // Get information on the right instruction and its operand - int right_inst; // Right-instructions's number + int right_inst; // Right-instruction's number if( isdigit(_curchar) ) { right_inst = get_int(); // Right-instruction's operand diff --git a/src/hotspot/share/adlc/adlparse.hpp b/src/hotspot/share/adlc/adlparse.hpp index 609c4565607..eddabcbb38d 100644 --- a/src/hotspot/share/adlc/adlparse.hpp +++ b/src/hotspot/share/adlc/adlparse.hpp @@ -62,7 +62,9 @@ class PipeDesc; class PipeClass; class RegList; // ***** Peephole Section ***** +class PeepPredicate; class PeepMatch; +class PeepProcedure; class PeepConstraint; class PeepReplace; @@ -136,7 +138,9 @@ protected: void pipe_class_parse(PipelineForm &pipe); // Parse pipeline class definition // Parse components of a peephole rule + void peep_predicate_parse(Peephole &peep); // Parse the peephole predicate void peep_match_parse(Peephole &peep); // Parse the peephole match rule + void peep_procedure_parse(Peephole &peep); // Parse the peephole procedure void peep_constraint_parse(Peephole &peep);// Parse the peephole constraints void peep_replace_parse(Peephole &peep); // Parse peephole replacement rule diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp index 6f1a5997511..93e902c7d6c 100644 --- a/src/hotspot/share/adlc/formsopt.cpp +++ b/src/hotspot/share/adlc/formsopt.cpp @@ -635,7 +635,8 @@ void PipeClassForm::output(FILE *fp) { // Write info to output files //==============================Peephole Optimization========================== int Peephole::_peephole_counter = 0; //------------------------------Peephole--------------------------------------- -Peephole::Peephole() : _match(NULL), _constraint(NULL), _replace(NULL), _next(NULL) { +Peephole::Peephole() : _predicate(NULL), _match(NULL), _procedure(NULL), + _constraint(NULL), _replace(NULL), _next(NULL) { _peephole_number = _peephole_counter++; } Peephole::~Peephole() { @@ -650,12 +651,24 @@ void Peephole::append_peephole(Peephole *next_peephole) { } } +// Add a predicate to this peephole rule +void Peephole::add_predicate(PeepPredicate* predicate) { + assert( _predicate == NULL, "fatal()" ); + _predicate = predicate; +} + // Store the components of this peephole rule void Peephole::add_match(PeepMatch *match) { assert( _match == NULL, "fatal()" ); _match = match; } +// Add a procedure to this peephole rule +void Peephole::add_procedure(PeepProcedure* procedure) { + assert( _procedure == NULL, "fatal()" ); + _procedure = procedure; +} + void Peephole::append_constraint(PeepConstraint *next_constraint) { if( _constraint == NULL ) { _constraint = next_constraint; @@ -685,13 +698,30 @@ void Peephole::output(FILE *fp) { // Write info to output files if( _next ) _next->output(fp); } +//----------------------------PeepPredicate------------------------------------ +PeepPredicate::PeepPredicate(const char* rule) : _rule(rule) { +} +PeepPredicate::~PeepPredicate() { +} + +const char* PeepPredicate::rule() const { + return _rule; +} + +void PeepPredicate::dump() { + output(stderr); +} + +void PeepPredicate::output(FILE* fp) { + fprintf(fp, "PeepPredicate\n"); +} + //------------------------------PeepMatch-------------------------------------- PeepMatch::PeepMatch(char *rule) : _max_position(0), _rule(rule) { } PeepMatch::~PeepMatch() { } - // Insert info into the match-rule void PeepMatch::add_instruction(int parent, int position, const char *name, int input) { @@ -741,6 +771,24 @@ void PeepMatch::output(FILE *fp) { // Write info to output files fprintf(fp,"PeepMatch:\n"); } +//----------------------------PeepProcedure------------------------------------ +PeepProcedure::PeepProcedure(const char* name) : _name(name) { +} +PeepProcedure::~PeepProcedure() { +} + +const char* PeepProcedure::name() const { + return _name; +} + +void PeepProcedure::dump() { + output(stderr); +} + +void PeepProcedure::output(FILE* fp) { + fprintf(fp, "PeepProcedure\n"); +} + //------------------------------PeepConstraint--------------------------------- PeepConstraint::PeepConstraint(int left_inst, char* left_op, char* relation, int right_inst, char* right_op) diff --git a/src/hotspot/share/adlc/formsopt.hpp b/src/hotspot/share/adlc/formsopt.hpp index 6e872b5c647..47027970ec4 100644 --- a/src/hotspot/share/adlc/formsopt.hpp +++ b/src/hotspot/share/adlc/formsopt.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,9 +47,6 @@ class ExpandRule; class RewriteRule; class ConstructRule; class FormatRule; -class Peephole; -class PeepMatch; -class PeepConstraint; class EncClass; class Interface; class RegInterface; @@ -67,7 +64,10 @@ class ResourceForm; class PipeClassForm; class PipeClassOperandForm; class PipeClassResourceForm; +class Peephole; +class PeepPredicate; class PeepMatch; +class PeepProcedure; class PeepConstraint; class PeepReplace; class MatchList; @@ -526,7 +526,9 @@ class Peephole : public Form { private: static int _peephole_counter;// Incremented by each peephole rule parsed int _peephole_number;// Remember my order in architecture description + PeepPredicate *_predicate; // Predicate to apply peep rule PeepMatch *_match; // Instruction pattern to match + PeepProcedure *_procedure; // The detailed procedure to perform the rule PeepConstraint *_constraint; // List of additional constraints PeepReplace *_replace; // Instruction pattern to substitute in @@ -541,13 +543,17 @@ public: void append_peephole(Peephole *next_peephole); // Store the components of this peephole rule + void add_predicate(PeepPredicate *only_one_predicate); void add_match(PeepMatch *only_one_match); + void add_procedure(PeepProcedure *only_one_procedure); void append_constraint(PeepConstraint *next_constraint); void add_replace(PeepReplace *only_one_replacement); // Access the components of this peephole rule int peephole_number() { return _peephole_number; } + PeepPredicate *predicate() { return _predicate; } PeepMatch *match() { return _match; } + PeepProcedure *procedure() { return _procedure; } PeepConstraint *constraints() { return _constraint; } PeepReplace *replacement() { return _replace; } Peephole *next() { return _next; } @@ -556,6 +562,19 @@ public: void output(FILE *fp); // Write info to output files }; +class PeepPredicate : public Form { +private: + const char* _rule; +public: + // Public Methods + PeepPredicate(const char* rule); + ~PeepPredicate(); + + const char* rule() const; + + void dump(); + void output(FILE* fp); +}; class PeepMatch : public Form { private: @@ -588,6 +607,19 @@ public: void output(FILE *fp); }; +class PeepProcedure : public Form { +private: + const char* _name; +public: + // Public Methods + PeepProcedure(const char* name); + ~PeepProcedure(); + + const char* name() const; + + void dump(); + void output(FILE* fp); +}; class PeepConstraint : public Form { private: diff --git a/src/hotspot/share/adlc/output_c.cpp b/src/hotspot/share/adlc/output_c.cpp index fd8f259cd51..264012b8f15 100644 --- a/src/hotspot/share/adlc/output_c.cpp +++ b/src/hotspot/share/adlc/output_c.cpp @@ -1265,7 +1265,8 @@ static void check_peepconstraints(FILE *fp, FormDict &globals, PeepMatch *pmatch // } // Construct the new sub-tree -static void generate_peepreplace( FILE *fp, FormDict &globals, PeepMatch *pmatch, PeepConstraint *pconstraint, PeepReplace *preplace, int max_position ) { +static void generate_peepreplace( FILE *fp, FormDict &globals, int peephole_number, PeepMatch *pmatch, + PeepConstraint *pconstraint, PeepReplace *preplace, int max_position ) { fprintf(fp, " // IF instructions and constraints matched\n"); fprintf(fp, " if( matches ) {\n"); fprintf(fp, " // generate the new sub-tree\n"); @@ -1314,7 +1315,6 @@ static void generate_peepreplace( FILE *fp, FormDict &globals, PeepMatch *pmatch fprintf(fp, " root->_bottom_type = inst%d->bottom_type();\n", inst_num); } // Define result register and result operand - fprintf(fp, " ra_->add_reference(root, inst%d);\n", inst_num); fprintf(fp, " ra_->set_oop (root, ra_->is_oop(inst%d));\n", inst_num); fprintf(fp, " ra_->set_pair(root->_idx, ra_->get_reg_second(inst%d), ra_->get_reg_first(inst%d));\n", inst_num, inst_num); fprintf(fp, " root->_opnds[0] = inst%d->_opnds[0]->clone(); // result\n", inst_num); @@ -1340,12 +1340,20 @@ static void generate_peepreplace( FILE *fp, FormDict &globals, PeepMatch *pmatch assert( false, "ShouldNotReachHere();"); } + // Set output of the new node + fprintf(fp, " inst0->replace_by(root);\n"); + // Mark the node as removed because peephole does not remove nodes from the graph for (int i = 0; i <= max_position; i++) { fprintf(fp, " inst%d->set_removed();\n", i); + fprintf(fp, " cfg_->map_node_to_block(inst%d, nullptr);\n", i); } - // Return the new sub-tree - fprintf(fp, " deleted = %d;\n", max_position+1 /*zero to one based*/); - fprintf(fp, " return root; // return new root;\n"); + for (int i = 0; i <= max_position; i++) { + fprintf(fp, " block->remove_node(block_index - %d);\n", i); + } + fprintf(fp, " block->insert_node(root, block_index - %d);\n", max_position); + fprintf(fp, " cfg_->map_node_to_block(root, block);\n"); + // Return the peephole index + fprintf(fp, " return %d; // return the peephole index;\n", peephole_number); fprintf(fp, " }\n"); } @@ -1353,7 +1361,7 @@ static void generate_peepreplace( FILE *fp, FormDict &globals, PeepMatch *pmatch // Define the Peephole method for an instruction node void ArchDesc::definePeephole(FILE *fp, InstructForm *node) { // Generate Peephole function header - fprintf(fp, "MachNode *%sNode::peephole(Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted) {\n", node->_ident); + fprintf(fp, "int %sNode::peephole(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_) {\n", node->_ident); fprintf(fp, " bool matches = true;\n"); // Identify the maximum instruction position, @@ -1366,6 +1374,9 @@ void ArchDesc::definePeephole(FILE *fp, InstructForm *node) { int max_position = 0; Peephole *peep; for( peep = node->peepholes(); peep != NULL; peep = peep->next() ) { + if (peep->procedure() != NULL) { + continue; + } PeepMatch *pmatch = peep->match(); assert( pmatch != NULL, "fatal(), missing peepmatch rule"); if( max_position < pmatch->max_position() ) max_position = pmatch->max_position(); @@ -1384,7 +1395,9 @@ void ArchDesc::definePeephole(FILE *fp, InstructForm *node) { // If these match, Generate the new subtree for( peep = node->peepholes(); peep != NULL; peep = peep->next() ) { int peephole_number = peep->peephole_number(); + PeepPredicate *ppredicate = peep->predicate(); PeepMatch *pmatch = peep->match(); + PeepProcedure *pprocedure = peep->procedure(); PeepConstraint *pconstraint = peep->constraints(); PeepReplace *preplace = peep->replacement(); @@ -1393,29 +1406,58 @@ void ArchDesc::definePeephole(FILE *fp, InstructForm *node) { "root of PeepMatch does not match instruction"); // Make each peephole rule individually selectable - fprintf(fp, " if( (OptoPeepholeAt == -1) || (OptoPeepholeAt==%d) ) {\n", peephole_number); - fprintf(fp, " matches = true;\n"); - // Scan the peepmatch and output a test for each instruction - check_peepmatch_instruction_sequence( fp, pmatch, pconstraint ); + fprintf(fp, " if( ((OptoPeepholeAt == -1) || (OptoPeepholeAt==%d)) && ( %s ) ) {\n", + peephole_number, ppredicate != NULL ? ppredicate->rule() : "true"); + if (pprocedure == NULL) { + fprintf(fp, " matches = true;\n"); + // Scan the peepmatch and output a test for each instruction + check_peepmatch_instruction_sequence( fp, pmatch, pconstraint ); - // Check constraints and build replacement inside scope - fprintf(fp, " // If instruction subtree matches\n"); - fprintf(fp, " if( matches ) {\n"); + // Check constraints and build replacement inside scope + fprintf(fp, " // If instruction subtree matches\n"); + fprintf(fp, " if( matches ) {\n"); - // Generate tests for the constraints - check_peepconstraints( fp, _globalNames, pmatch, pconstraint ); + // Generate tests for the constraints + check_peepconstraints( fp, _globalNames, pmatch, pconstraint ); - // Construct the new sub-tree - generate_peepreplace( fp, _globalNames, pmatch, pconstraint, preplace, max_position ); + // Construct the new sub-tree + generate_peepreplace( fp, _globalNames, peephole_number, pmatch, pconstraint, preplace, max_position ); + + // End of scope for this peephole's constraints + fprintf(fp, " }\n"); + } else { + const char* replace_inst = NULL; + preplace->next_instruction(replace_inst); + // Generate the target instruction + fprintf(fp, " auto replacing = [](){ return static_cast(new %sNode()); };\n", replace_inst); + + // Call the precedure + fprintf(fp, " bool replacement = Peephole::%s(block, block_index, cfg_, ra_, replacing", pprocedure->name()); + + int parent = -1; + int inst_position = 0; + const char* inst_name = NULL; + int input = 0; + pmatch->reset(); + for (pmatch->next_instruction(parent, inst_position, inst_name, input); + inst_name != NULL; + pmatch->next_instruction(parent, inst_position, inst_name, input)) { + fprintf(fp, ", %s_rule", inst_name); + } + fprintf(fp, ");\n"); + + // If substitution succeeded, return the new node + fprintf(fp, " if (replacement) {\n"); + fprintf(fp, " return %d;\n", peephole_number); + fprintf(fp, " }\n"); + } - // End of scope for this peephole's constraints - fprintf(fp, " }\n"); // Closing brace '}' to make each peephole rule individually selectable fprintf(fp, " } // end of peephole rule #%d\n", peephole_number); fprintf(fp, "\n"); } - fprintf(fp, " return NULL; // No peephole rules matched\n"); + fprintf(fp, " return -1; // No peephole rules matched\n"); fprintf(fp, "}\n"); fprintf(fp, "\n"); } diff --git a/src/hotspot/share/adlc/output_h.cpp b/src/hotspot/share/adlc/output_h.cpp index 3528298b261..029bbb63034 100644 --- a/src/hotspot/share/adlc/output_h.cpp +++ b/src/hotspot/share/adlc/output_h.cpp @@ -1703,7 +1703,7 @@ void ArchDesc::declareClasses(FILE *fp) { // If there is an explicit peephole rule, build it if ( instr->peepholes() != NULL ) { - fprintf(fp," virtual MachNode *peephole(Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted);\n"); + fprintf(fp," virtual int peephole(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_);\n"); } // Output the declaration for number of relocation entries diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index bccccc7b6e3..faa45ef3934 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -429,8 +429,8 @@ int MachNode::operand_index(Node* def) const { //------------------------------peephole--------------------------------------- // Apply peephole rule(s) to this instruction -MachNode *MachNode::peephole(Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted) { - return NULL; +int MachNode::peephole(Block *block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc *ra_) { + return -1; } //------------------------------add_case_label--------------------------------- diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index c94b1ec079e..921cf158ee9 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -359,7 +359,7 @@ public: virtual const class TypePtr *adr_type() const; // Apply peephole rule(s) to this instruction - virtual MachNode *peephole(Block *block, int block_index, PhaseRegAlloc *ra_, int &deleted); + virtual int peephole(Block *block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc *ra_); // Top-level ideal Opcode matched virtual int ideal_Opcode() const { return Op_Node; } diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 98f5379f364..4a071c84fdf 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -2161,52 +2161,39 @@ void PhasePeephole::do_transform() { Block* block = _cfg.get_block(block_number); bool block_not_printed = true; - // and each instruction within a block - uint end_index = block->number_of_nodes(); - // block->end_idx() not valid after PhaseRegAlloc - for( uint instruction_index = 1; instruction_index < end_index; ++instruction_index ) { - Node *n = block->get_node(instruction_index); - if( n->is_Mach() ) { - MachNode *m = n->as_Mach(); - int deleted_count = 0; - // check for peephole opportunities - MachNode *m2 = m->peephole(block, instruction_index, _regalloc, deleted_count); - if( m2 != NULL ) { + for (bool progress = true; progress;) { + progress = false; + // block->end_idx() not valid after PhaseRegAlloc + uint end_index = block->number_of_nodes(); + for( uint instruction_index = end_index - 1; instruction_index > 0; --instruction_index ) { + Node *n = block->get_node(instruction_index); + if( n->is_Mach() ) { + MachNode *m = n->as_Mach(); + // check for peephole opportunities + int result = m->peephole(block, instruction_index, &_cfg, _regalloc); + if( result != -1 ) { #ifndef PRODUCT - if( PrintOptoPeephole ) { - // Print method, first time only - if( C->method() && method_name_not_printed ) { - C->method()->print_short_name(); tty->cr(); - method_name_not_printed = false; + if( PrintOptoPeephole ) { + // Print method, first time only + if( C->method() && method_name_not_printed ) { + C->method()->print_short_name(); tty->cr(); + method_name_not_printed = false; + } + // Print this block + if( Verbose && block_not_printed) { + tty->print_cr("in block"); + block->dump(); + block_not_printed = false; + } + // Print the peephole number + tty->print_cr("peephole number: %d", result); } - // Print this block - if( Verbose && block_not_printed) { - tty->print_cr("in block"); - block->dump(); - block_not_printed = false; - } - // Print instructions being deleted - for( int i = (deleted_count - 1); i >= 0; --i ) { - block->get_node(instruction_index-i)->as_Mach()->format(_regalloc); tty->cr(); - } - tty->print_cr("replaced with"); - // Print new instruction - m2->format(_regalloc); - tty->print("\n\n"); - } + inc_peepholes(); #endif - // Remove old nodes from basic block and update instruction_index - // (old nodes still exist and may have edges pointing to them - // as register allocation info is stored in the allocator using - // the node index to live range mappings.) - uint safe_instruction_index = (instruction_index - deleted_count); - for( ; (instruction_index > safe_instruction_index); --instruction_index ) { - block->remove_node( instruction_index ); + // Set progress, start again + progress = true; + break; } - // install new node after safe_instruction_index - block->insert_node(m2, safe_instruction_index + 1); - end_index = block->number_of_nodes() - 1; // Recompute new block size - NOT_PRODUCT( inc_peepholes(); ) } } } diff --git a/test/micro/org/openjdk/bench/vm/compiler/x86/LeaPeephole.java b/test/micro/org/openjdk/bench/vm/compiler/x86/LeaPeephole.java new file mode 100644 index 00000000000..a2dd51f9742 --- /dev/null +++ b/test/micro/org/openjdk/bench/vm/compiler/x86/LeaPeephole.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.bench.vm.compiler.x86; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(value = 1) +@State(Scope.Thread) +public class LeaPeephole { + static final int ITERATION = 1000; + + int x, y; + + @Benchmark + public void B_I_int(Blackhole bh) { + int x = this.x; + int y = this.y; + for (int i = 0; i < ITERATION; i++) { + int x1 = x + y; + x = x1 + y; + y = x1 + x; + } + bh.consume(x); + bh.consume(y); + } + + @Benchmark + public void B_D_int(Blackhole bh) { + int x = this.x; + int y = this.y; + for (int i = 0; i < ITERATION; i++) { + bh.consume(x + 10); + bh.consume(x + 20); + bh.consume(x + 30); + bh.consume(y + 10); + bh.consume(y + 20); + bh.consume(y + 30); + x = x >> 1; + y = y >> 2; + } + } + + @Benchmark + public void I_S_int(Blackhole bh) { + int x = this.x; + int y = this.y; + for (int i = 0; i < ITERATION; i++) { + bh.consume(x << 1); + bh.consume(x << 2); + bh.consume(x << 3); + bh.consume(y << 1); + bh.consume(y << 2); + bh.consume(y << 3); + x = x >> 1; + y = y >> 2; + } + } + + @Benchmark + public void B_I_long(Blackhole bh) { + long x = this.x; + long y = this.y; + for (int i = 0; i < ITERATION; i++) { + long x1 = x + y; + x = x1 + y; + y = x1 + x; + } + bh.consume(x); + bh.consume(y); + } + + @Benchmark + public void B_D_long(Blackhole bh) { + long x = this.x; + long y = this.y; + for (int i = 0; i < ITERATION; i++) { + bh.consume(x + 10); + bh.consume(x + 20); + bh.consume(x + 30); + bh.consume(y + 10); + bh.consume(y + 20); + bh.consume(y + 30); + x = x >> 1; + y = y >> 2; + } + } + + @Benchmark + public void I_S_long(Blackhole bh) { + long x = this.x; + long y = this.y; + for (int i = 0; i < ITERATION; i++) { + bh.consume(x << 1); + bh.consume(x << 2); + bh.consume(x << 3); + bh.consume(y << 1); + bh.consume(y << 2); + bh.consume(y << 3); + x = x >> 1; + y = y >> 2; + } + } +}