Archive

Archive for the ‘java 0 day analysis’ Category

CVE-2013-0809 Analysis

April 12, 2013 Leave a comment

There are lots of fix in the code, should be self explantory:

     1.1 --- a/src/share/classes/sun/awt/image/ByteComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     1.2 +++ b/src/share/classes/sun/awt/image/ByteComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     1.3 @@ -868,6 +868,15 @@
     1.4       * or if data buffer has not enough capacity.
     1.5       */
     1.6      protected final void verify() {
     1.7 +        /* Need to re-verify the dimensions since a sample model may be
     1.8 +         * specified to the constructor
     1.9 +         */
    1.10 +        if (width <= 0 || height <= 0 ||
    1.11 +            height > (Integer.MAX_VALUE / width))
    1.12 +        {
    1.13 +            throw new RasterFormatException("Invalid raster dimension");
    1.14 +        }
    1.15 +
    1.16          for (int i = 0; i < dataOffsets.length; i++) {
    1.17              if (dataOffsets[i] < 0) {
    1.18                  throw new RasterFormatException("Data offsets for band " + i
    1.19 @@ -905,13 +914,14 @@
    1.20          lastPixelOffset += lastScanOffset;
    1.21  
    1.22          for (int i = 0; i < numDataElements; i++) {
    1.23 -            size = lastPixelOffset + dataOffsets[i];
    1.24              if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    1.25                  throw new RasterFormatException("Incorrect band offset: "
    1.26                              + dataOffsets[i]);
    1.27  
    1.28              }
    1.29  
    1.30 +            size = lastPixelOffset + dataOffsets[i];
    1.31 +
    1.32              if (size > maxSize) {
    1.33                  maxSize = size;
    1.34              }
     2.1 --- a/src/share/classes/sun/awt/image/BytePackedRaster.java	Tue Dec 04 14:02:08 2012 +0000
     2.2 +++ b/src/share/classes/sun/awt/image/BytePackedRaster.java	Thu Feb 14 19:51:51 2013 +0400
     2.3 @@ -1368,11 +1368,35 @@
     2.4              throw new RasterFormatException("Data offsets must be >= 0");
     2.5          }
     2.6  
     2.7 +        /* Need to re-verify the dimensions since a sample model may be
     2.8 +         * specified to the constructor
     2.9 +         */
    2.10 +        if (width <= 0 || height <= 0 ||
    2.11 +            height > (Integer.MAX_VALUE / width))
    2.12 +        {
    2.13 +            throw new RasterFormatException("Invalid raster dimension");
    2.14 +        }
    2.15 +
    2.16 +
    2.17 +        /*
    2.18 +         * pixelBitstride was verified in constructor, so just make
    2.19 +         * sure that it is safe to multiply it by width.
    2.20 +         */
    2.21 +        if ((width - 1) > Integer.MAX_VALUE / pixelBitStride) {
    2.22 +            throw new RasterFormatException("Invalid raster dimension");
    2.23 +        }
    2.24 +
    2.25 +        if (scanlineStride < 0 ||
    2.26 +            scanlineStride > (Integer.MAX_VALUE / height))
    2.27 +        {
    2.28 +            throw new RasterFormatException("Invalid scanline stride");
    2.29 +        }
    2.30 +
    2.31          int lastbit = (dataBitOffset
    2.32                         + (height-1) * scanlineStride * 8
    2.33                         + (width-1) * pixelBitStride
    2.34                         + pixelBitStride - 1);
    2.35 -        if (lastbit / 8 >= data.length) {
    2.36 +        if (lastbit < 0 || lastbit / 8 >= data.length) {
    2.37              throw new RasterFormatException("raster dimensions overflow " +
    2.38                                              "array bounds");
    2.39          }
     3.1 --- a/src/share/classes/sun/awt/image/IntegerComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     3.2 +++ b/src/share/classes/sun/awt/image/IntegerComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     3.3 @@ -208,7 +208,7 @@
     3.4                                              " SinglePixelPackedSampleModel");
     3.5          }
     3.6  
     3.7 -        verify(false);
     3.8 +        verify();
     3.9      }
    3.10  
    3.11  
    3.12 @@ -629,16 +629,26 @@
    3.13      }
    3.14  
    3.15      /**
    3.16 -     * Verify that the layout parameters are consistent with
    3.17 -     * the data.  If strictCheck
    3.18 -     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
    3.19 -     * strictCheck is true, this method will check for additional error
    3.20 -     * conditions such as line wraparound (width of a line greater than
    3.21 -     * the scanline stride).
    3.22 -     * @return   String   Error string, if the layout is incompatible with
    3.23 -     *                    the data.  Otherwise returns null.
    3.24 +     * Verify that the layout parameters are consistent with the data.
    3.25 +     *
    3.26 +     * The method verifies whether scanline stride and pixel stride do not
    3.27 +     * cause an integer overflow during calculation of a position of the pixel
    3.28 +     * in data buffer. It also verifies whether the data buffer has enough data
    3.29 +     *  to correspond the raster layout attributes.
    3.30 +     *
    3.31 +     * @throws RasterFormatException if an integer overflow is detected,
    3.32 +     * or if data buffer has not enough capacity.
    3.33       */
    3.34 -    private void verify (boolean strictCheck) {
    3.35 +    protected final void verify() {
    3.36 +        /* Need to re-verify the dimensions since a sample model may be
    3.37 +         * specified to the constructor
    3.38 +         */
    3.39 +        if (width <= 0 || height <= 0 ||
    3.40 +            height > (Integer.MAX_VALUE / width))
    3.41 +        {
    3.42 +            throw new RasterFormatException("Invalid raster dimension");
    3.43 +        }
    3.44 +
    3.45          if (dataOffsets[0] < 0) {
    3.46              throw new RasterFormatException("Data offset ("+dataOffsets[0]+
    3.47                                              ") must be >= 0");
    3.48 @@ -647,17 +657,46 @@
    3.49          int maxSize = 0;
    3.50          int size;
    3.51  
    3.52 -        for (int i=0; i < numDataElements; i++) {
    3.53 -            size = (height-1)*scanlineStride + (width-1)*pixelStride +
    3.54 -                dataOffsets[i];
    3.55 +        // we can be sure that width and height are greater than 0
    3.56 +        if (scanlineStride < 0 ||
    3.57 +            scanlineStride > (Integer.MAX_VALUE / height))
    3.58 +        {
    3.59 +            // integer overflow
    3.60 +            throw new RasterFormatException("Incorrect scanline stride: "
    3.61 +                    + scanlineStride);
    3.62 +        }
    3.63 +        int lastScanOffset = (height - 1) * scanlineStride;
    3.64 +
    3.65 +        if (pixelStride < 0 ||
    3.66 +            pixelStride > (Integer.MAX_VALUE / width))
    3.67 +        {
    3.68 +            // integer overflow
    3.69 +            throw new RasterFormatException("Incorrect pixel stride: "
    3.70 +                    + pixelStride);
    3.71 +        }
    3.72 +        int lastPixelOffset = (width - 1) * pixelStride;
    3.73 +
    3.74 +        if (lastPixelOffset > (Integer.MAX_VALUE - lastScanOffset)) {
    3.75 +            // integer overflow
    3.76 +            throw new RasterFormatException("Incorrect raster attributes");
    3.77 +        }
    3.78 +        lastPixelOffset += lastScanOffset;
    3.79 +
    3.80 +        for (int i = 0; i < numDataElements; i++) {
    3.81 +            if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    3.82 +                throw new RasterFormatException("Incorrect band offset: "
    3.83 +                            + dataOffsets[i]);
    3.84 +            }
    3.85 +
    3.86 +            size = lastPixelOffset + dataOffsets[i];
    3.87 +
    3.88              if (size > maxSize) {
    3.89                  maxSize = size;
    3.90              }
    3.91          }
    3.92          if (data.length < maxSize) {
    3.93 -            throw new RasterFormatException("Data array too small (should be "+
    3.94 -                                          maxSize
    3.95 -                                          +" but is "+data.length+" )");
    3.96 +            throw new RasterFormatException("Data array too small (should be "
    3.97 +                    + maxSize + " )");
    3.98          }
    3.99      }
   3.100
     4.1 --- a/src/share/classes/sun/awt/image/IntegerInterleavedRaster.java	Tue Dec 04 14:02:08 2012 +0000
     4.2 +++ b/src/share/classes/sun/awt/image/IntegerInterleavedRaster.java	Thu Feb 14 19:51:51 2013 +0400
     4.3 @@ -151,7 +151,7 @@
     4.4              throw new RasterFormatException("IntegerInterleavedRasters must have"+
     4.5                                              " SinglePixelPackedSampleModel");
     4.6          }
     4.7 -        verify(false);
     4.8 +        verify();
     4.9      }
    4.10  
    4.11  
    4.12 @@ -540,31 +540,6 @@
    4.13          return createCompatibleWritableRaster(width,height);
    4.14      }
    4.15  
    4.16 -    /**
    4.17 -     * Verify that the layout parameters are consistent with
    4.18 -     * the data.  If strictCheck
    4.19 -     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
    4.20 -     * strictCheck is true, this method will check for additional error
    4.21 -     * conditions such as line wraparound (width of a line greater than
    4.22 -     * the scanline stride).
    4.23 -     * @return   String   Error string, if the layout is incompatible with
    4.24 -     *                    the data.  Otherwise returns null.
    4.25 -     */
    4.26 -    private void verify (boolean strictCheck) {
    4.27 -        int maxSize = 0;
    4.28 -        int size;
    4.29 -
    4.30 -        size = (height-1)*scanlineStride + (width-1) + dataOffsets[0];
    4.31 -        if (size > maxSize) {
    4.32 -            maxSize = size;
    4.33 -        }
    4.34 -        if (data.length < maxSize) {
    4.35 -            throw new RasterFormatException("Data array too small (should be "+
    4.36 -                                          maxSize
    4.37 -                                          +" but is "+data.length+" )");
    4.38 -        }
    4.39 -    }
    4.40 -
    4.41      public String toString() {
    4.42          return new String ("IntegerInterleavedRaster: width = "+width
    4.43                             +" height = " + height
     5.1 --- a/src/share/classes/sun/awt/image/ShortComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     5.2 +++ b/src/share/classes/sun/awt/image/ShortComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     5.3 @@ -802,6 +802,15 @@
     5.4       * or if data buffer has not enough capacity.
     5.5       */
     5.6      protected final void verify() {
     5.7 +        /* Need to re-verify the dimensions since a sample model may be
     5.8 +         * specified to the constructor
     5.9 +         */
    5.10 +        if (width <= 0 || height <= 0 ||
    5.11 +            height > (Integer.MAX_VALUE / width))
    5.12 +        {
    5.13 +            throw new RasterFormatException("Invalid raster dimension");
    5.14 +        }
    5.15 +
    5.16          for (int i = 0; i < dataOffsets.length; i++) {
    5.17              if (dataOffsets[i] < 0) {
    5.18                  throw new RasterFormatException("Data offsets for band " + i
    5.19 @@ -839,12 +848,13 @@
    5.20          lastPixelOffset += lastScanOffset;
    5.21  
    5.22          for (int i = 0; i < numDataElements; i++) {
    5.23 -            size = lastPixelOffset + dataOffsets[i];
    5.24              if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    5.25                  throw new RasterFormatException("Incorrect band offset: "
    5.26                              + dataOffsets[i]);
    5.27              }
    5.28  
    5.29 +            size = lastPixelOffset + dataOffsets[i];
    5.30 +
    5.31              if (size > maxSize) {
    5.32                  maxSize = size;
    5.33              }
     6.1 --- a/src/share/native/sun/awt/image/awt_parseImage.c	Tue Dec 04 14:02:08 2012 +0000
     6.2 +++ b/src/share/native/sun/awt/image/awt_parseImage.c	Thu Feb 14 19:51:51 2013 +0400
     6.3 @@ -34,6 +34,7 @@
     6.4  #include "java_awt_color_ColorSpace.h"
     6.5  #include "awt_Mlib.h"
     6.6  #include "safe_alloc.h"
     6.7 +#include "safe_math.h"
     6.8  
     6.9  static int setHints(JNIEnv *env, BufImageS_t *imageP);
    6.10
     7.1 --- a/src/share/native/sun/awt/medialib/awt_ImagingLib.c	Tue Dec 04 14:02:08 2012 +0000
     7.2 +++ b/src/share/native/sun/awt/medialib/awt_ImagingLib.c	Thu Feb 14 19:51:51 2013 +0400
     7.3 @@ -42,6 +42,7 @@
     7.4  #include "awt_Mlib.h"
     7.5  #include "gdefs.h"
     7.6  #include "safe_alloc.h"
     7.7 +#include "safe_math.h"
     7.8  
     7.9  /***************************************************************************
    7.10   *                               Definitions                               *
    7.11 @@ -1993,13 +1994,23 @@
    7.12      unsigned char *dP = dataP;
    7.13  #define NUM_LINES    10
    7.14      int numLines = NUM_LINES;
    7.15 -    int nbytes = rasterP->width*4*NUM_LINES;
    7.16 +    /* it is safe to calculate the scan length, because width has been verified
    7.17 +     * on creation of the mlib image
    7.18 +     */
    7.19 +    int scanLength = rasterP->width * 4;
    7.20 +
    7.21 +    int nbytes = 0;
    7.22 +    if (!SAFE_TO_MULT(numLines, scanLength)) {
    7.23 +        return -1;
    7.24 +    }
    7.25 +
    7.26 +    nbytes = numLines * scanLength;
    7.27  
    7.28      for (y=0; y < rasterP->height; y+=numLines) {
    7.29          /* getData, one scanline at a time */
    7.30          if (y+numLines > rasterP->height) {
    7.31              numLines = rasterP->height - y;
    7.32 -            nbytes = rasterP->width*4*numLines;
    7.33 +            nbytes = numLines * scanLength;
    7.34          }
    7.35          jpixels = (*env)->CallObjectMethod(env, imageP->jimage,
    7.36                                             g_BImgGetRGBMID, 0, y,
    7.37 @@ -2129,8 +2140,14 @@
    7.38      if (cvtToDefault) {
    7.39          int status = 0;
    7.40          *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, 4, width, height);
    7.41 +        if (*mlibImagePP == NULL) {
    7.42 +            return -1;
    7.43 +        }
    7.44          cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.45 -        /* Make sure the image is cleared */
    7.46 +        /* Make sure the image is cleared.
    7.47 +         * NB: the image dimension is already verified, so we can
    7.48 +         * safely calculate the length of the buffer.
    7.49 +         */
    7.50          memset(cDataP, 0, width*height*4);
    7.51  
    7.52          if (!isSrc) {
    7.53 @@ -2380,6 +2397,9 @@
    7.54      case sun_awt_image_IntegerComponentRaster_TYPE_BYTE_PACKED_SAMPLES:
    7.55          *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.56                                          width, height);
    7.57 +        if (*mlibImagePP == NULL) {
    7.58 +            return -1;
    7.59 +        }
    7.60          if (!isSrc) return 0;
    7.61          cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.62          return expandPackedBCR(env, rasterP, -1, cDataP);
    7.63 @@ -2388,6 +2408,9 @@
    7.64          if (rasterP->sppsm.maxBitSize <= 8) {
    7.65              *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.66                                              width, height);
    7.67 +            if (*mlibImagePP == NULL) {
    7.68 +                return -1;
    7.69 +            }
    7.70              if (!isSrc) return 0;
    7.71              cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.72              return expandPackedSCR(env, rasterP, -1, cDataP);
    7.73 @@ -2397,6 +2420,9 @@
    7.74          if (rasterP->sppsm.maxBitSize <= 8) {
    7.75              *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.76                                              width, height);
    7.77 +            if (*mlibImagePP == NULL) {
    7.78 +                return -1;
    7.79 +            }
    7.80              if (!isSrc) return 0;
    7.81              cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.82              return expandPackedICR(env, rasterP, -1, cDataP);
     8.1 --- a/src/share/native/sun/awt/medialib/mlib_ImageCreate.c	Tue Dec 04 14:02:08 2012 +0000
     8.2 +++ b/src/share/native/sun/awt/medialib/mlib_ImageCreate.c	Thu Feb 14 19:51:51 2013 +0400
     8.3 @@ -120,6 +120,7 @@
     8.4  #include "mlib_image.h"
     8.5  #include "mlib_ImageRowTable.h"
     8.6  #include "mlib_ImageCreate.h"
     8.7 +#include "safe_math.h"
     8.8  
     8.9  /***************************************************************/
    8.10  mlib_image* mlib_ImageSet(mlib_image *image,
    8.11 @@ -247,28 +248,50 @@
    8.12      return NULL;
    8.13    };
    8.14  
    8.15 +  if (!SAFE_TO_MULT(width, channels)) {
    8.16 +    return NULL;
    8.17 +  }
    8.18 +
    8.19 +  wb = width * channels;
    8.20 +
    8.21    switch (type) {
    8.22      case MLIB_DOUBLE:
    8.23 -      wb = width * channels * 8;
    8.24 +      if (!SAFE_TO_MULT(wb, 8)) {
    8.25 +        return NULL;
    8.26 +      }
    8.27 +      wb *= 8;
    8.28        break;
    8.29      case MLIB_FLOAT:
    8.30      case MLIB_INT:
    8.31 -      wb = width * channels * 4;
    8.32 +      if (!SAFE_TO_MULT(wb, 4)) {
    8.33 +        return NULL;
    8.34 +      }
    8.35 +      wb *= 4;
    8.36        break;
    8.37      case MLIB_USHORT:
    8.38      case MLIB_SHORT:
    8.39 -      wb = width * channels * 2;
    8.40 +      if (!SAFE_TO_MULT(wb, 4)) {
    8.41 +        return NULL;
    8.42 +      }
    8.43 +      wb *= 2;
    8.44        break;
    8.45      case MLIB_BYTE:
    8.46 -      wb = width * channels;
    8.47 +      // wb is ready
    8.48        break;
    8.49      case MLIB_BIT:
    8.50 -      wb = (width * channels + 7) / 8;
    8.51 +      if (!SAFE_TO_ADD(7, wb)) {
    8.52 +        return NULL;
    8.53 +      }
    8.54 +      wb = (wb + 7) / 8;
    8.55        break;
    8.56      default:
    8.57        return NULL;
    8.58    }
    8.59  
    8.60 +  if (!SAFE_TO_MULT(wb, height)) {
    8.61 +      return NULL;
    8.62 +  }
    8.63 +
    8.64    data = mlib_malloc(wb * height);
    8.65    if (data == NULL) {
    8.66      return NULL;
     9.1 --- a/src/share/native/sun/awt/medialib/safe_alloc.h	Tue Dec 04 14:02:08 2012 +0000
     9.2 +++ b/src/share/native/sun/awt/medialib/safe_alloc.h	Thu Feb 14 19:51:51 2013 +0400
     9.3 @@ -41,10 +41,4 @@
     9.4      (((w) > 0) && ((h) > 0) && ((sz) > 0) &&                               \
     9.5       (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > ((juint)(sz))))
     9.6  
     9.7 -#define SAFE_TO_MULT(a, b) \
     9.8 -    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
     9.9 -
    9.10 -#define SAFE_TO_ADD(a, b) \
    9.11 -    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
    9.12 -
    9.13  #endif // __SAFE_ALLOC_H__
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/src/share/native/sun/awt/medialib/safe_math.h	Thu Feb 14 19:51:51 2013 +0400
    10.3 @@ -0,0 +1,35 @@
    10.4 +/*
    10.5 + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
    10.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    10.7 + *
    10.8 + * This code is free software; you can redistribute it and/or modify it
    10.9 + * under the terms of the GNU General Public License version 2 only, as
   10.10 + * published by the Free Software Foundation.  Oracle designates this
   10.11 + * particular file as subject to the "Classpath" exception as provided
   10.12 + * by Oracle in the LICENSE file that accompanied this code.
   10.13 + *
   10.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
   10.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   10.16 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   10.17 + * version 2 for more details (a copy is included in the LICENSE file that
   10.18 + * accompanied this code).
   10.19 + *
   10.20 + * You should have received a copy of the GNU General Public License version
   10.21 + * 2 along with this work; if not, write to the Free Software Foundation,
   10.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   10.23 + *
   10.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   10.25 + * or visit www.oracle.com if you need additional information or have any
   10.26 + * questions.
   10.27 + */
   10.28 +
   10.29 +#ifndef __SAFE_MATH_H__
   10.30 +#define __SAFE_MATH_H__
   10.31 +
   10.32 +#define SAFE_TO_MULT(a, b) \
   10.33 +    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
   10.34 +
   10.35 +#define SAFE_TO_ADD(a, b) \
   10.36 +    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
   10.37 +
   10.38 +#endif // __SAFE_MATH_H__

CVE-2013-1493 analysis.

March 21, 2013 Leave a comment

I have recived the sample from a friend of mine. some of them are highly obfuscated and some are not. when i tested them with my latest java version, they were causing jvm to crash. so i decided to digg further. first thing i did was decompiled the jar file and modified the code to print debug statments so that i can trace out and understand the code while debugging. then i removed uncessessary code and compiled and ran the code in eclipse. then i see following:

ScreenHunter_50 Mar. 21 23.34

on further debugging i come to know that this is crashing in icctransform.colorconvert function. if you check the code you can see that this function is defined here:

http://docs.oracle.com/javase/1.5.0/docs/api/java/awt/image/ColorConvertOp.html

“This class performs a pixel-by-pixel color conversion of the data in the source image. The resulting color values are scaled to the precision of the destination image. Color conversion can be specified via an array of ColorSpace objects or an array of ICC_Profile objects.

If the source is a BufferedImage with premultiplied alpha, the color components are divided by the alpha component before color conversion. If the destination is a BufferedImage with premultiplied alpha, the color components are multiplied by the alpha component after conversion. Rasters are treated as having no alpha channel, i.e. all bands are color bands.

If a RenderingHints object is specified in the constructor, the color rendering hint and the dithering hint may be used to control color conversion.

Note that Source and Destination may be the same object.”

it was crashing on making following call from the jar file:

ScreenHunter_51 Mar. 21 23.39

so there might be some kind of overflow. when i attached debugger, i see that its crashing inside cmm.dll .

if we check the code there are some values which are more then integer.MAX size:

ScreenHunter_52 Mar. 21 23.40

if you check in the above image, doffsets has a very large value which is 50000000.

This can be confirmed from the patch from open jdk from url:

http://icedtea.classpath.org/hg/release/icedtea7-forest-2.3/jdk/rev/4f97a6256473

line diff
     1.1 --- a/src/share/classes/sun/java2d/cmm/lcms/LCMSImageLayout.java	Thu Feb 14 19:51:51 2013 +0400
     1.2 +++ b/src/share/classes/sun/java2d/cmm/lcms/LCMSImageLayout.java	Thu Feb 21 11:25:43 2013 +0400
     1.3 @@ -99,50 +99,75 @@
     1.4      int offset;
     1.5  
     1.6      Object dataArray;
     1.7 -    private LCMSImageLayout(int np, int pixelType, int pixelSize) {
     1.8 +    private int dataArrayLength; /* in bytes */
     1.9 +
    1.10 +    private LCMSImageLayout(int np, int pixelType, int pixelSize)
    1.11 +            throws ImageLayoutException
    1.12 +    {
    1.13          this.pixelType = pixelType;
    1.14          width = np;
    1.15          height = 1;
    1.16 -        nextRowOffset = np*pixelSize;
    1.17 +        nextRowOffset = safeMult(pixelSize, np);
    1.18          offset = 0;
    1.19      }
    1.20  
    1.21      private LCMSImageLayout(int width, int height, int pixelType,
    1.22 -                            int pixelSize) {
    1.23 +                            int pixelSize)
    1.24 +            throws ImageLayoutException
    1.25 +    {
    1.26          this.pixelType = pixelType;
    1.27          this.width = width;
    1.28          this.height = height;
    1.29 -        nextRowOffset = width*pixelSize;
    1.30 +        nextRowOffset = safeMult(pixelSize, width);
    1.31          offset = 0;
    1.32      }
    1.33  
    1.34  
    1.35 -    public LCMSImageLayout(byte[] data, int np, int pixelType, int pixelSize) {
    1.36 +    public LCMSImageLayout(byte[] data, int np, int pixelType, int pixelSize)
    1.37 +            throws ImageLayoutException
    1.38 +    {
    1.39          this(np, pixelType, pixelSize);
    1.40          dataType = DT_BYTE;
    1.41          dataArray = data;
    1.42 +        dataArrayLength = data.length;
    1.43 +
    1.44 +        verify();
    1.45      }
    1.46  
    1.47 -    public LCMSImageLayout(short[] data, int np, int pixelType, int pixelSize) {
    1.48 +    public LCMSImageLayout(short[] data, int np, int pixelType, int pixelSize)
    1.49 +            throws ImageLayoutException
    1.50 +    {
    1.51          this(np, pixelType, pixelSize);
    1.52          dataType = DT_SHORT;
    1.53          dataArray = data;
    1.54 +        dataArrayLength = 2 * data.length;
    1.55 +
    1.56 +        verify();
    1.57      }
    1.58  
    1.59 -    public LCMSImageLayout(int[] data, int np, int pixelType, int pixelSize) {
    1.60 +    public LCMSImageLayout(int[] data, int np, int pixelType, int pixelSize)
    1.61 +            throws ImageLayoutException
    1.62 +    {
    1.63          this(np, pixelType, pixelSize);
    1.64          dataType = DT_INT;
    1.65          dataArray = data;
    1.66 +        dataArrayLength = 4 * data.length;
    1.67 +
    1.68 +        verify();
    1.69      }
    1.70  
    1.71      public LCMSImageLayout(double[] data, int np, int pixelType, int pixelSize)
    1.72 +            throws ImageLayoutException
    1.73      {
    1.74          this(np, pixelType, pixelSize);
    1.75          dataType = DT_DOUBLE;
    1.76          dataArray = data;
    1.77 +        dataArrayLength = 8 * data.length;
    1.78 +
    1.79 +        verify();
    1.80      }
    1.81  
    1.82 -    public LCMSImageLayout(BufferedImage image) {
    1.83 +    public LCMSImageLayout(BufferedImage image) throws ImageLayoutException {
    1.84          ShortComponentRaster shortRaster;
    1.85          IntegerComponentRaster intRaster;
    1.86          ByteComponentRaster byteRaster;
    1.87 @@ -186,9 +211,13 @@
    1.88              case BufferedImage.TYPE_INT_ARGB:
    1.89              case BufferedImage.TYPE_INT_BGR:
    1.90                  intRaster = (IntegerComponentRaster)image.getRaster();
    1.91 -                nextRowOffset = intRaster.getScanlineStride()*4;
    1.92 -                offset = intRaster.getDataOffset(0)*4;
    1.93 +
    1.94 +                nextRowOffset = safeMult(4, intRaster.getScanlineStride());
    1.95 +
    1.96 +                offset = safeMult(4, intRaster.getDataOffset(0));
    1.97 +
    1.98                  dataArray = intRaster.getDataStorage();
    1.99 +                dataArrayLength = 4 * intRaster.getDataStorage().length;
   1.100                  dataType = DT_INT;
   1.101                  break;
   1.102  
   1.103 @@ -196,8 +225,10 @@
   1.104              case BufferedImage.TYPE_4BYTE_ABGR:
   1.105                  byteRaster = (ByteComponentRaster)image.getRaster();
   1.106                  nextRowOffset = byteRaster.getScanlineStride();
   1.107 -                offset = byteRaster.getDataOffset(0);
   1.108 +                int firstBand = image.getSampleModel().getNumBands() - 1;
   1.109 +                offset = byteRaster.getDataOffset(firstBand);
   1.110                  dataArray = byteRaster.getDataStorage();
   1.111 +                dataArrayLength = byteRaster.getDataStorage().length;
   1.112                  dataType = DT_BYTE;
   1.113                  break;
   1.114  
   1.115 @@ -206,17 +237,20 @@
   1.116                  nextRowOffset = byteRaster.getScanlineStride();
   1.117                  offset = byteRaster.getDataOffset(0);
   1.118                  dataArray = byteRaster.getDataStorage();
   1.119 +                dataArrayLength = byteRaster.getDataStorage().length;
   1.120                  dataType = DT_BYTE;
   1.121                  break;
   1.122  
   1.123              case BufferedImage.TYPE_USHORT_GRAY:
   1.124                  shortRaster = (ShortComponentRaster)image.getRaster();
   1.125 -                nextRowOffset = shortRaster.getScanlineStride()*2;
   1.126 -                offset = shortRaster.getDataOffset(0) * 2;
   1.127 +                nextRowOffset = safeMult(2, shortRaster.getScanlineStride());
   1.128 +                offset = safeMult(2, shortRaster.getDataOffset(0));
   1.129                  dataArray = shortRaster.getDataStorage();
   1.130 +                dataArrayLength = 2 * shortRaster.getDataStorage().length;
   1.131                  dataType = DT_SHORT;
   1.132                  break;
   1.133          }
   1.134 +        verify();
   1.135      }
   1.136  
   1.137      public static boolean isSupported(BufferedImage image) {
   1.138 @@ -232,4 +266,45 @@
   1.139          }
   1.140          return false;
   1.141      }
   1.142 +
   1.143 +    private void verify() throws ImageLayoutException {
   1.144 +
   1.145 +        if (offset < 0 || offset >= dataArrayLength) {
   1.146 +            throw new ImageLayoutException("Invalid image layout");
   1.147 +        }
   1.148 +
   1.149 +        int lastPixelOffset = safeMult(nextRowOffset, (height - 1));
   1.150 +
   1.151 +        lastPixelOffset = safeAdd(lastPixelOffset, (width - 1));
   1.152 +
   1.153 +        int off = safeAdd(offset, lastPixelOffset);
   1.154 +
   1.155 +        if (off < 0 || off >= dataArrayLength) {
   1.156 +            throw new ImageLayoutException("Invalid image layout");
   1.157 +        }
   1.158 +    }
   1.159 +
   1.160 +    static int safeAdd(int a, int b) throws ImageLayoutException {
   1.161 +        long res = a;
   1.162 +        res += b;
   1.163 +        if (res < Integer.MIN_VALUE || res > Integer.MAX_VALUE) {
   1.164 +            throw new ImageLayoutException("Invalid image layout");
   1.165 +        }
   1.166 +        return (int)res;
   1.167 +    }
   1.168 +
   1.169 +    static int safeMult(int a, int b) throws ImageLayoutException {
   1.170 +        long res = a;
   1.171 +        res *= b;
   1.172 +        if (res < Integer.MIN_VALUE || res > Integer.MAX_VALUE) {
   1.173 +            throw new ImageLayoutException("Invalid image layout");
   1.174 +        }
   1.175 +        return (int)res;
   1.176 +    }
   1.177 +
   1.178 +    public static class ImageLayoutException extends Exception {
   1.179 +        public ImageLayoutException(String message) {
   1.180 +            super(message);
   1.181 +        }
   1.182 +    }
   1.183  }
     2.1 --- a/src/share/classes/sun/java2d/cmm/lcms/LCMSTransform.java	Thu Feb 14 19:51:51 2013 +0400
     2.2 +++ b/src/share/classes/sun/java2d/cmm/lcms/LCMSTransform.java	Thu Feb 21 11:25:43 2013 +0400
     2.3 @@ -51,6 +51,7 @@
     2.4  import java.awt.image.ComponentSampleModel;
     2.5  import sun.java2d.cmm.*;
     2.6  import sun.java2d.cmm.lcms.*;
     2.7 +import static sun.java2d.cmm.lcms.LCMSImageLayout.ImageLayoutException;
     2.8  
     2.9  
    2.10  public class LCMSTransform implements ColorTransform {
    2.11 @@ -157,8 +158,12 @@
    2.12          if (LCMSImageLayout.isSupported(src) &&
    2.13              LCMSImageLayout.isSupported(dst))
    2.14          {
    2.15 -            doTransform(new LCMSImageLayout(src), new LCMSImageLayout(dst));
    2.16 -            return;
    2.17 +            try {
    2.18 +                doTransform(new LCMSImageLayout(src), new LCMSImageLayout(dst));
    2.19 +                return;
    2.20 +            } catch (ImageLayoutException e) {
    2.21 +                throw new CMMException("Unable to convert images");
    2.22 +            }
    2.23          }
    2.24          LCMSImageLayout srcIL, dstIL;
    2.25          Raster srcRas = src.getRaster();
    2.26 @@ -216,14 +221,18 @@
    2.27              }
    2.28              int idx;
    2.29              // TODO check for src npixels = dst npixels
    2.30 -            srcIL = new LCMSImageLayout(
    2.31 -                srcLine, srcLine.length/getNumInComponents(),
    2.32 -                LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.33 -                LCMSImageLayout.BYTES_SH(1), getNumInComponents());
    2.34 -            dstIL = new LCMSImageLayout(
    2.35 -                dstLine, dstLine.length/getNumOutComponents(),
    2.36 -                LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
    2.37 -                LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
    2.38 +            try {
    2.39 +                srcIL = new LCMSImageLayout(
    2.40 +                        srcLine, srcLine.length/getNumInComponents(),
    2.41 +                        LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.42 +                        LCMSImageLayout.BYTES_SH(1), getNumInComponents());
    2.43 +                dstIL = new LCMSImageLayout(
    2.44 +                        dstLine, dstLine.length/getNumOutComponents(),
    2.45 +                        LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
    2.46 +                        LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
    2.47 +            } catch (ImageLayoutException e) {
    2.48 +                throw new CMMException("Unable to convert images");
    2.49 +            }
    2.50              // process each scanline
    2.51              for (int y = 0; y < h; y++) {
    2.52                  // convert src scanline
    2.53 @@ -272,16 +281,19 @@
    2.54                  alpha = new float[w];
    2.55              }
    2.56              int idx;
    2.57 -            srcIL = new LCMSImageLayout(
    2.58 -                srcLine, srcLine.length/getNumInComponents(),
    2.59 -                LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.60 -                LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
    2.61 +            try {
    2.62 +                srcIL = new LCMSImageLayout(
    2.63 +                    srcLine, srcLine.length/getNumInComponents(),
    2.64 +                    LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.65 +                    LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
    2.66  
    2.67 -            dstIL = new LCMSImageLayout(
    2.68 -                dstLine, dstLine.length/getNumOutComponents(),
    2.69 -                LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
    2.70 -                LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
    2.71 -
    2.72 +                dstIL = new LCMSImageLayout(
    2.73 +                    dstLine, dstLine.length/getNumOutComponents(),
    2.74 +                    LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
    2.75 +                    LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
    2.76 +            } catch (ImageLayoutException e) {
    2.77 +                throw new CMMException("Unable to convert images");
    2.78 +            }
    2.79              // process each scanline
    2.80              for (int y = 0; y < h; y++) {
    2.81                  // convert src scanline
    2.82 @@ -390,16 +402,19 @@
    2.83          short[] srcLine = new short[w * srcNumBands];
    2.84          short[] dstLine = new short[w * dstNumBands];
    2.85          int idx;
    2.86 -        srcIL = new LCMSImageLayout(
    2.87 -            srcLine, srcLine.length/getNumInComponents(),
    2.88 -            LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.89 -            LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
    2.90 +        try {
    2.91 +            srcIL = new LCMSImageLayout(
    2.92 +                    srcLine, srcLine.length/getNumInComponents(),
    2.93 +                    LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
    2.94 +                    LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
    2.95  
    2.96 -        dstIL = new LCMSImageLayout(
    2.97 -            dstLine, dstLine.length/getNumOutComponents(),
    2.98 -            LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
    2.99 -            LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.100 -
   2.101 +            dstIL = new LCMSImageLayout(
   2.102 +                    dstLine, dstLine.length/getNumOutComponents(),
   2.103 +                    LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.104 +                    LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.105 +        } catch (ImageLayoutException e) {
   2.106 +            throw new CMMException("Unable to convert rasters");
   2.107 +        }
   2.108          // process each scanline
   2.109          for (int y = 0; y < h; y++, ys++, yd++) {
   2.110              // get src scanline
   2.111 @@ -482,15 +497,18 @@
   2.112              byte[] dstLine = new byte[w * dstNumBands];
   2.113              int idx;
   2.114              // TODO check for src npixels = dst npixels
   2.115 -            srcIL = new LCMSImageLayout(
   2.116 -                srcLine, srcLine.length/getNumInComponents(),
   2.117 -                LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.118 -                LCMSImageLayout.BYTES_SH(1), getNumInComponents());
   2.119 -            dstIL = new LCMSImageLayout(
   2.120 -                dstLine, dstLine.length/getNumOutComponents(),
   2.121 -                LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.122 -                LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
   2.123 -
   2.124 +            try {
   2.125 +                srcIL = new LCMSImageLayout(
   2.126 +                        srcLine, srcLine.length/getNumInComponents(),
   2.127 +                        LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.128 +                        LCMSImageLayout.BYTES_SH(1), getNumInComponents());
   2.129 +                dstIL = new LCMSImageLayout(
   2.130 +                        dstLine, dstLine.length/getNumOutComponents(),
   2.131 +                        LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.132 +                        LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
   2.133 +            } catch (ImageLayoutException e) {
   2.134 +                throw new CMMException("Unable to convert rasters");
   2.135 +            }
   2.136              // process each scanline
   2.137              for (int y = 0; y < h; y++, ys++, yd++) {
   2.138                  // get src scanline
   2.139 @@ -522,16 +540,20 @@
   2.140              short[] srcLine = new short[w * srcNumBands];
   2.141              short[] dstLine = new short[w * dstNumBands];
   2.142              int idx;
   2.143 -            srcIL = new LCMSImageLayout(
   2.144 -                srcLine, srcLine.length/getNumInComponents(),
   2.145 -                LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.146 -                LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
   2.147  
   2.148 -            dstIL = new LCMSImageLayout(
   2.149 -                dstLine, dstLine.length/getNumOutComponents(),
   2.150 -                LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.151 -                LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.152 +            try {
   2.153 +                srcIL = new LCMSImageLayout(
   2.154 +                        srcLine, srcLine.length/getNumInComponents(),
   2.155 +                        LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.156 +                        LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
   2.157  
   2.158 +                dstIL = new LCMSImageLayout(
   2.159 +                        dstLine, dstLine.length/getNumOutComponents(),
   2.160 +                        LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.161 +                        LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.162 +            } catch (ImageLayoutException e) {
   2.163 +                throw new CMMException("Unable to convert rasters");
   2.164 +            }
   2.165              // process each scanline
   2.166              for (int y = 0; y < h; y++, ys++, yd++) {
   2.167                  // get src scanline
   2.168 @@ -572,19 +594,23 @@
   2.169              dst = new short [(src.length/getNumInComponents())*getNumOutComponents()];
   2.170          }
   2.171  
   2.172 -        LCMSImageLayout srcIL = new LCMSImageLayout(
   2.173 -            src, src.length/getNumInComponents(),
   2.174 -            LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.175 -            LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
   2.176 +        try {
   2.177 +            LCMSImageLayout srcIL = new LCMSImageLayout(
   2.178 +                    src, src.length/getNumInComponents(),
   2.179 +                    LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.180 +                    LCMSImageLayout.BYTES_SH(2), getNumInComponents()*2);
   2.181  
   2.182 -        LCMSImageLayout dstIL = new LCMSImageLayout(
   2.183 -            dst, dst.length/getNumOutComponents(),
   2.184 -            LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.185 -            LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.186 +            LCMSImageLayout dstIL = new LCMSImageLayout(
   2.187 +                    dst, dst.length/getNumOutComponents(),
   2.188 +                    LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.189 +                    LCMSImageLayout.BYTES_SH(2), getNumOutComponents()*2);
   2.190  
   2.191 -        doTransform(srcIL, dstIL);
   2.192 +            doTransform(srcIL, dstIL);
   2.193  
   2.194 -        return dst;
   2.195 +            return dst;
   2.196 +        } catch (ImageLayoutException e) {
   2.197 +            throw new CMMException("Unable to convert data");
   2.198 +        }
   2.199      }
   2.200  
   2.201      public byte[] colorConvert(byte[] src, byte[] dst) {
   2.202 @@ -592,18 +618,22 @@
   2.203              dst = new byte [(src.length/getNumInComponents())*getNumOutComponents()];
   2.204          }
   2.205  
   2.206 -        LCMSImageLayout srcIL = new LCMSImageLayout(
   2.207 -            src, src.length/getNumInComponents(),
   2.208 -            LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.209 -            LCMSImageLayout.BYTES_SH(1), getNumInComponents());
   2.210 +        try {
   2.211 +            LCMSImageLayout srcIL = new LCMSImageLayout(
   2.212 +                    src, src.length/getNumInComponents(),
   2.213 +                    LCMSImageLayout.CHANNELS_SH(getNumInComponents()) |
   2.214 +                    LCMSImageLayout.BYTES_SH(1), getNumInComponents());
   2.215  
   2.216 -        LCMSImageLayout dstIL = new LCMSImageLayout(
   2.217 -            dst, dst.length/getNumOutComponents(),
   2.218 -            LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.219 -            LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
   2.220 +            LCMSImageLayout dstIL = new LCMSImageLayout(
   2.221 +                    dst, dst.length/getNumOutComponents(),
   2.222 +                    LCMSImageLayout.CHANNELS_SH(getNumOutComponents()) |
   2.223 +                    LCMSImageLayout.BYTES_SH(1), getNumOutComponents());
   2.224  
   2.225 -        doTransform(srcIL, dstIL);
   2.226 +            doTransform(srcIL, dstIL);
   2.227  
   2.228 -        return dst;
   2.229 +            return dst;
   2.230 +        } catch (ImageLayoutException e) {
   2.231 +            throw new CMMException("Unable to convert data");
   2.232 +        }
   2.233      }
   2.234  }

thanks,
Hardik
%d bloggers like this: