8204196: integer cleanup

Reviewed-by: xuelei

src/java.base/share/classes/com/sun/crypto/provider/RSACipher.java

@@ -332,7 +332,7 @@ public final class RSACipher extends CipherSpi {
         if ((inLen == 0) || (in == null)) {
             return;
         }
-        if (bufOfs + inLen > buffer.length) {
+        if (inLen > (buffer.length - bufOfs)) {
             bufOfs = buffer.length + 1;
             return;
         }

src/java.base/share/classes/javax/crypto/Cipher.java

@@ -2739,7 +2739,7 @@ public class Cipher {
 
         // Input sanity check
         if ((src == null) || (offset < 0) || (len < 0)
-                || ((len + offset) > src.length)) {
+                || len > (src.length - offset)) {
             throw new IllegalArgumentException("Bad arguments");
         }
 

src/java.base/share/classes/sun/security/provider/DSA.java

@@ -588,7 +588,7 @@ abstract class DSA extends SignatureSpi {
                 }
             }
             protected void engineUpdate(byte[] input, int offset, int len) {
-                if (ofs + len > digestBuffer.length) {
+                if (len > (digestBuffer.length - ofs)) {
                     ofs = Integer.MAX_VALUE;
                 } else {
                     System.arraycopy(input, offset, digestBuffer, ofs, len);
@@ -597,7 +597,7 @@ abstract class DSA extends SignatureSpi {
             }
             protected final void engineUpdate(ByteBuffer input) {
                 int inputLen = input.remaining();
-                if (ofs + inputLen > digestBuffer.length) {
+                if (inputLen > (digestBuffer.length - ofs)) {
                     ofs = Integer.MAX_VALUE;
                 } else {
                     input.get(digestBuffer, ofs, inputLen);

src/java.base/share/classes/sun/security/x509/AlgorithmId.java

@@ -1039,7 +1039,7 @@ public class AlgorithmId implements Serializable, DerEncoder {
      * @return the default alg, might be null if unsupported
      */
     public static String getDefaultSigAlgForKey(PrivateKey k) {
-        switch (k.getAlgorithm().toUpperCase(Locale.ROOT)) {
+        switch (k.getAlgorithm().toUpperCase(Locale.ENGLISH)) {
             case "EC":
                 return ecStrength(KeyUtil.getKeySize(k))
                     + "withECDSA";

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java

@@ -507,6 +507,10 @@ final class P11Signature extends SignatureSpi {
         if (len == 0) {
             return;
         }
+        // check for overflow
+        if (len + bytesProcessed < 0) {
+            throw new ProviderException("Processed bytes limits exceeded.");
+        }
         switch (type) {
         case T_UPDATE:
             try {

src/jdk.crypto.mscapi/windows/classes/sun/security/mscapi/RSASignature.java

@@ -129,7 +129,7 @@ abstract class RSASignature extends java.security.SignatureSpi
         @Override
         protected void engineUpdate(byte[] b, int off, int len)
                 throws SignatureException {
-            if (offset + len > precomputedDigest.length) {
+            if (len > (precomputedDigest.length - offset)) {
                 offset = RAW_RSA_MAX + 1;
                 return;
             }
@@ -144,7 +144,7 @@ abstract class RSASignature extends java.security.SignatureSpi
             if (len <= 0) {
                 return;
             }
-            if (offset + len > precomputedDigest.length) {
+            if (len > (precomputedDigest.length - offset)) {
                 offset = RAW_RSA_MAX + 1;
                 return;
             }

src/jdk.crypto.ucrypto/solaris/classes/com/oracle/security/ucrypto/NativeRSASignature.java

@@ -291,8 +291,9 @@ class NativeRSASignature extends SignatureSpi {
         throws SignatureException {
         boolean doCancel = true;
         try {
-            if (outbuf == null || (offset < 0) || (outbuf.length < (offset + sigLength))
-                || (len < sigLength)) {
+            if (outbuf == null || (offset < 0) ||
+                    ((outbuf.length - offset) < sigLength) ||
+                    (len < sigLength)) {
                 throw new SignatureException("Invalid output buffer. offset: " +
                     offset + ". len: " + len + ". sigLength: " + sigLength);
             }
@@ -357,8 +358,9 @@ class NativeRSASignature extends SignatureSpi {
         throws SignatureException {
         boolean doCancel = true;
         try {
-            if (sigBytes == null || (sigOfs < 0) || (sigBytes.length < (sigOfs + this.sigLength))
-                || (sigLen != this.sigLength)) {
+            if (sigBytes == null || (sigOfs < 0) ||
+                    ((sigBytes.length - sigOfs) < this.sigLength) ||
+                    (sigLen != this.sigLength)) {
                 throw new SignatureException("Invalid signature length: got " +
                     sigLen + " but was expecting " + this.sigLength);
             }
@@ -440,7 +442,7 @@ class NativeRSASignature extends SignatureSpi {
 
     // returns 0 (success) or negative (ucrypto error occurred)
     private int update(byte[] in, int inOfs, int inLen) {
-        if (inOfs < 0 || inOfs + inLen > in.length) {
+        if (inOfs < 0 || inOfs > (in.length - inLen)) {
             throw new ArrayIndexOutOfBoundsException("inOfs :" + inOfs +
                 ". inLen: " + inLen + ". in.length: " + in.length);
         }