Archive

Posts Tagged ‘cve-2013-0809’

CVE-2013-0809 Analysis

April 12, 2013 Leave a comment

There are lots of fix in the code, should be self explantory:

     1.1 --- a/src/share/classes/sun/awt/image/ByteComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     1.2 +++ b/src/share/classes/sun/awt/image/ByteComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     1.3 @@ -868,6 +868,15 @@
     1.4       * or if data buffer has not enough capacity.
     1.5       */
     1.6      protected final void verify() {
     1.7 +        /* Need to re-verify the dimensions since a sample model may be
     1.8 +         * specified to the constructor
     1.9 +         */
    1.10 +        if (width <= 0 || height <= 0 ||
    1.11 +            height > (Integer.MAX_VALUE / width))
    1.12 +        {
    1.13 +            throw new RasterFormatException("Invalid raster dimension");
    1.14 +        }
    1.15 +
    1.16          for (int i = 0; i < dataOffsets.length; i++) {
    1.17              if (dataOffsets[i] < 0) {
    1.18                  throw new RasterFormatException("Data offsets for band " + i
    1.19 @@ -905,13 +914,14 @@
    1.20          lastPixelOffset += lastScanOffset;
    1.21  
    1.22          for (int i = 0; i < numDataElements; i++) {
    1.23 -            size = lastPixelOffset + dataOffsets[i];
    1.24              if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    1.25                  throw new RasterFormatException("Incorrect band offset: "
    1.26                              + dataOffsets[i]);
    1.27  
    1.28              }
    1.29  
    1.30 +            size = lastPixelOffset + dataOffsets[i];
    1.31 +
    1.32              if (size > maxSize) {
    1.33                  maxSize = size;
    1.34              }
     2.1 --- a/src/share/classes/sun/awt/image/BytePackedRaster.java	Tue Dec 04 14:02:08 2012 +0000
     2.2 +++ b/src/share/classes/sun/awt/image/BytePackedRaster.java	Thu Feb 14 19:51:51 2013 +0400
     2.3 @@ -1368,11 +1368,35 @@
     2.4              throw new RasterFormatException("Data offsets must be >= 0");
     2.5          }
     2.6  
     2.7 +        /* Need to re-verify the dimensions since a sample model may be
     2.8 +         * specified to the constructor
     2.9 +         */
    2.10 +        if (width <= 0 || height <= 0 ||
    2.11 +            height > (Integer.MAX_VALUE / width))
    2.12 +        {
    2.13 +            throw new RasterFormatException("Invalid raster dimension");
    2.14 +        }
    2.15 +
    2.16 +
    2.17 +        /*
    2.18 +         * pixelBitstride was verified in constructor, so just make
    2.19 +         * sure that it is safe to multiply it by width.
    2.20 +         */
    2.21 +        if ((width - 1) > Integer.MAX_VALUE / pixelBitStride) {
    2.22 +            throw new RasterFormatException("Invalid raster dimension");
    2.23 +        }
    2.24 +
    2.25 +        if (scanlineStride < 0 ||
    2.26 +            scanlineStride > (Integer.MAX_VALUE / height))
    2.27 +        {
    2.28 +            throw new RasterFormatException("Invalid scanline stride");
    2.29 +        }
    2.30 +
    2.31          int lastbit = (dataBitOffset
    2.32                         + (height-1) * scanlineStride * 8
    2.33                         + (width-1) * pixelBitStride
    2.34                         + pixelBitStride - 1);
    2.35 -        if (lastbit / 8 >= data.length) {
    2.36 +        if (lastbit < 0 || lastbit / 8 >= data.length) {
    2.37              throw new RasterFormatException("raster dimensions overflow " +
    2.38                                              "array bounds");
    2.39          }
     3.1 --- a/src/share/classes/sun/awt/image/IntegerComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     3.2 +++ b/src/share/classes/sun/awt/image/IntegerComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     3.3 @@ -208,7 +208,7 @@
     3.4                                              " SinglePixelPackedSampleModel");
     3.5          }
     3.6  
     3.7 -        verify(false);
     3.8 +        verify();
     3.9      }
    3.10  
    3.11  
    3.12 @@ -629,16 +629,26 @@
    3.13      }
    3.14  
    3.15      /**
    3.16 -     * Verify that the layout parameters are consistent with
    3.17 -     * the data.  If strictCheck
    3.18 -     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
    3.19 -     * strictCheck is true, this method will check for additional error
    3.20 -     * conditions such as line wraparound (width of a line greater than
    3.21 -     * the scanline stride).
    3.22 -     * @return   String   Error string, if the layout is incompatible with
    3.23 -     *                    the data.  Otherwise returns null.
    3.24 +     * Verify that the layout parameters are consistent with the data.
    3.25 +     *
    3.26 +     * The method verifies whether scanline stride and pixel stride do not
    3.27 +     * cause an integer overflow during calculation of a position of the pixel
    3.28 +     * in data buffer. It also verifies whether the data buffer has enough data
    3.29 +     *  to correspond the raster layout attributes.
    3.30 +     *
    3.31 +     * @throws RasterFormatException if an integer overflow is detected,
    3.32 +     * or if data buffer has not enough capacity.
    3.33       */
    3.34 -    private void verify (boolean strictCheck) {
    3.35 +    protected final void verify() {
    3.36 +        /* Need to re-verify the dimensions since a sample model may be
    3.37 +         * specified to the constructor
    3.38 +         */
    3.39 +        if (width <= 0 || height <= 0 ||
    3.40 +            height > (Integer.MAX_VALUE / width))
    3.41 +        {
    3.42 +            throw new RasterFormatException("Invalid raster dimension");
    3.43 +        }
    3.44 +
    3.45          if (dataOffsets[0] < 0) {
    3.46              throw new RasterFormatException("Data offset ("+dataOffsets[0]+
    3.47                                              ") must be >= 0");
    3.48 @@ -647,17 +657,46 @@
    3.49          int maxSize = 0;
    3.50          int size;
    3.51  
    3.52 -        for (int i=0; i < numDataElements; i++) {
    3.53 -            size = (height-1)*scanlineStride + (width-1)*pixelStride +
    3.54 -                dataOffsets[i];
    3.55 +        // we can be sure that width and height are greater than 0
    3.56 +        if (scanlineStride < 0 ||
    3.57 +            scanlineStride > (Integer.MAX_VALUE / height))
    3.58 +        {
    3.59 +            // integer overflow
    3.60 +            throw new RasterFormatException("Incorrect scanline stride: "
    3.61 +                    + scanlineStride);
    3.62 +        }
    3.63 +        int lastScanOffset = (height - 1) * scanlineStride;
    3.64 +
    3.65 +        if (pixelStride < 0 ||
    3.66 +            pixelStride > (Integer.MAX_VALUE / width))
    3.67 +        {
    3.68 +            // integer overflow
    3.69 +            throw new RasterFormatException("Incorrect pixel stride: "
    3.70 +                    + pixelStride);
    3.71 +        }
    3.72 +        int lastPixelOffset = (width - 1) * pixelStride;
    3.73 +
    3.74 +        if (lastPixelOffset > (Integer.MAX_VALUE - lastScanOffset)) {
    3.75 +            // integer overflow
    3.76 +            throw new RasterFormatException("Incorrect raster attributes");
    3.77 +        }
    3.78 +        lastPixelOffset += lastScanOffset;
    3.79 +
    3.80 +        for (int i = 0; i < numDataElements; i++) {
    3.81 +            if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    3.82 +                throw new RasterFormatException("Incorrect band offset: "
    3.83 +                            + dataOffsets[i]);
    3.84 +            }
    3.85 +
    3.86 +            size = lastPixelOffset + dataOffsets[i];
    3.87 +
    3.88              if (size > maxSize) {
    3.89                  maxSize = size;
    3.90              }
    3.91          }
    3.92          if (data.length < maxSize) {
    3.93 -            throw new RasterFormatException("Data array too small (should be "+
    3.94 -                                          maxSize
    3.95 -                                          +" but is "+data.length+" )");
    3.96 +            throw new RasterFormatException("Data array too small (should be "
    3.97 +                    + maxSize + " )");
    3.98          }
    3.99      }
   3.100
     4.1 --- a/src/share/classes/sun/awt/image/IntegerInterleavedRaster.java	Tue Dec 04 14:02:08 2012 +0000
     4.2 +++ b/src/share/classes/sun/awt/image/IntegerInterleavedRaster.java	Thu Feb 14 19:51:51 2013 +0400
     4.3 @@ -151,7 +151,7 @@
     4.4              throw new RasterFormatException("IntegerInterleavedRasters must have"+
     4.5                                              " SinglePixelPackedSampleModel");
     4.6          }
     4.7 -        verify(false);
     4.8 +        verify();
     4.9      }
    4.10  
    4.11  
    4.12 @@ -540,31 +540,6 @@
    4.13          return createCompatibleWritableRaster(width,height);
    4.14      }
    4.15  
    4.16 -    /**
    4.17 -     * Verify that the layout parameters are consistent with
    4.18 -     * the data.  If strictCheck
    4.19 -     * is false, this method will check for ArrayIndexOutOfBounds conditions.  If
    4.20 -     * strictCheck is true, this method will check for additional error
    4.21 -     * conditions such as line wraparound (width of a line greater than
    4.22 -     * the scanline stride).
    4.23 -     * @return   String   Error string, if the layout is incompatible with
    4.24 -     *                    the data.  Otherwise returns null.
    4.25 -     */
    4.26 -    private void verify (boolean strictCheck) {
    4.27 -        int maxSize = 0;
    4.28 -        int size;
    4.29 -
    4.30 -        size = (height-1)*scanlineStride + (width-1) + dataOffsets[0];
    4.31 -        if (size > maxSize) {
    4.32 -            maxSize = size;
    4.33 -        }
    4.34 -        if (data.length < maxSize) {
    4.35 -            throw new RasterFormatException("Data array too small (should be "+
    4.36 -                                          maxSize
    4.37 -                                          +" but is "+data.length+" )");
    4.38 -        }
    4.39 -    }
    4.40 -
    4.41      public String toString() {
    4.42          return new String ("IntegerInterleavedRaster: width = "+width
    4.43                             +" height = " + height
     5.1 --- a/src/share/classes/sun/awt/image/ShortComponentRaster.java	Tue Dec 04 14:02:08 2012 +0000
     5.2 +++ b/src/share/classes/sun/awt/image/ShortComponentRaster.java	Thu Feb 14 19:51:51 2013 +0400
     5.3 @@ -802,6 +802,15 @@
     5.4       * or if data buffer has not enough capacity.
     5.5       */
     5.6      protected final void verify() {
     5.7 +        /* Need to re-verify the dimensions since a sample model may be
     5.8 +         * specified to the constructor
     5.9 +         */
    5.10 +        if (width <= 0 || height <= 0 ||
    5.11 +            height > (Integer.MAX_VALUE / width))
    5.12 +        {
    5.13 +            throw new RasterFormatException("Invalid raster dimension");
    5.14 +        }
    5.15 +
    5.16          for (int i = 0; i < dataOffsets.length; i++) {
    5.17              if (dataOffsets[i] < 0) {
    5.18                  throw new RasterFormatException("Data offsets for band " + i
    5.19 @@ -839,12 +848,13 @@
    5.20          lastPixelOffset += lastScanOffset;
    5.21  
    5.22          for (int i = 0; i < numDataElements; i++) {
    5.23 -            size = lastPixelOffset + dataOffsets[i];
    5.24              if (dataOffsets[i] > (Integer.MAX_VALUE - lastPixelOffset)) {
    5.25                  throw new RasterFormatException("Incorrect band offset: "
    5.26                              + dataOffsets[i]);
    5.27              }
    5.28  
    5.29 +            size = lastPixelOffset + dataOffsets[i];
    5.30 +
    5.31              if (size > maxSize) {
    5.32                  maxSize = size;
    5.33              }
     6.1 --- a/src/share/native/sun/awt/image/awt_parseImage.c	Tue Dec 04 14:02:08 2012 +0000
     6.2 +++ b/src/share/native/sun/awt/image/awt_parseImage.c	Thu Feb 14 19:51:51 2013 +0400
     6.3 @@ -34,6 +34,7 @@
     6.4  #include "java_awt_color_ColorSpace.h"
     6.5  #include "awt_Mlib.h"
     6.6  #include "safe_alloc.h"
     6.7 +#include "safe_math.h"
     6.8  
     6.9  static int setHints(JNIEnv *env, BufImageS_t *imageP);
    6.10
     7.1 --- a/src/share/native/sun/awt/medialib/awt_ImagingLib.c	Tue Dec 04 14:02:08 2012 +0000
     7.2 +++ b/src/share/native/sun/awt/medialib/awt_ImagingLib.c	Thu Feb 14 19:51:51 2013 +0400
     7.3 @@ -42,6 +42,7 @@
     7.4  #include "awt_Mlib.h"
     7.5  #include "gdefs.h"
     7.6  #include "safe_alloc.h"
     7.7 +#include "safe_math.h"
     7.8  
     7.9  /***************************************************************************
    7.10   *                               Definitions                               *
    7.11 @@ -1993,13 +1994,23 @@
    7.12      unsigned char *dP = dataP;
    7.13  #define NUM_LINES    10
    7.14      int numLines = NUM_LINES;
    7.15 -    int nbytes = rasterP->width*4*NUM_LINES;
    7.16 +    /* it is safe to calculate the scan length, because width has been verified
    7.17 +     * on creation of the mlib image
    7.18 +     */
    7.19 +    int scanLength = rasterP->width * 4;
    7.20 +
    7.21 +    int nbytes = 0;
    7.22 +    if (!SAFE_TO_MULT(numLines, scanLength)) {
    7.23 +        return -1;
    7.24 +    }
    7.25 +
    7.26 +    nbytes = numLines * scanLength;
    7.27  
    7.28      for (y=0; y < rasterP->height; y+=numLines) {
    7.29          /* getData, one scanline at a time */
    7.30          if (y+numLines > rasterP->height) {
    7.31              numLines = rasterP->height - y;
    7.32 -            nbytes = rasterP->width*4*numLines;
    7.33 +            nbytes = numLines * scanLength;
    7.34          }
    7.35          jpixels = (*env)->CallObjectMethod(env, imageP->jimage,
    7.36                                             g_BImgGetRGBMID, 0, y,
    7.37 @@ -2129,8 +2140,14 @@
    7.38      if (cvtToDefault) {
    7.39          int status = 0;
    7.40          *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, 4, width, height);
    7.41 +        if (*mlibImagePP == NULL) {
    7.42 +            return -1;
    7.43 +        }
    7.44          cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.45 -        /* Make sure the image is cleared */
    7.46 +        /* Make sure the image is cleared.
    7.47 +         * NB: the image dimension is already verified, so we can
    7.48 +         * safely calculate the length of the buffer.
    7.49 +         */
    7.50          memset(cDataP, 0, width*height*4);
    7.51  
    7.52          if (!isSrc) {
    7.53 @@ -2380,6 +2397,9 @@
    7.54      case sun_awt_image_IntegerComponentRaster_TYPE_BYTE_PACKED_SAMPLES:
    7.55          *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.56                                          width, height);
    7.57 +        if (*mlibImagePP == NULL) {
    7.58 +            return -1;
    7.59 +        }
    7.60          if (!isSrc) return 0;
    7.61          cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.62          return expandPackedBCR(env, rasterP, -1, cDataP);
    7.63 @@ -2388,6 +2408,9 @@
    7.64          if (rasterP->sppsm.maxBitSize <= 8) {
    7.65              *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.66                                              width, height);
    7.67 +            if (*mlibImagePP == NULL) {
    7.68 +                return -1;
    7.69 +            }
    7.70              if (!isSrc) return 0;
    7.71              cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.72              return expandPackedSCR(env, rasterP, -1, cDataP);
    7.73 @@ -2397,6 +2420,9 @@
    7.74          if (rasterP->sppsm.maxBitSize <= 8) {
    7.75              *mlibImagePP = (*sMlibSysFns.createFP)(MLIB_BYTE, rasterP->numBands,
    7.76                                              width, height);
    7.77 +            if (*mlibImagePP == NULL) {
    7.78 +                return -1;
    7.79 +            }
    7.80              if (!isSrc) return 0;
    7.81              cDataP  = (unsigned char *) mlib_ImageGetData(*mlibImagePP);
    7.82              return expandPackedICR(env, rasterP, -1, cDataP);
     8.1 --- a/src/share/native/sun/awt/medialib/mlib_ImageCreate.c	Tue Dec 04 14:02:08 2012 +0000
     8.2 +++ b/src/share/native/sun/awt/medialib/mlib_ImageCreate.c	Thu Feb 14 19:51:51 2013 +0400
     8.3 @@ -120,6 +120,7 @@
     8.4  #include "mlib_image.h"
     8.5  #include "mlib_ImageRowTable.h"
     8.6  #include "mlib_ImageCreate.h"
     8.7 +#include "safe_math.h"
     8.8  
     8.9  /***************************************************************/
    8.10  mlib_image* mlib_ImageSet(mlib_image *image,
    8.11 @@ -247,28 +248,50 @@
    8.12      return NULL;
    8.13    };
    8.14  
    8.15 +  if (!SAFE_TO_MULT(width, channels)) {
    8.16 +    return NULL;
    8.17 +  }
    8.18 +
    8.19 +  wb = width * channels;
    8.20 +
    8.21    switch (type) {
    8.22      case MLIB_DOUBLE:
    8.23 -      wb = width * channels * 8;
    8.24 +      if (!SAFE_TO_MULT(wb, 8)) {
    8.25 +        return NULL;
    8.26 +      }
    8.27 +      wb *= 8;
    8.28        break;
    8.29      case MLIB_FLOAT:
    8.30      case MLIB_INT:
    8.31 -      wb = width * channels * 4;
    8.32 +      if (!SAFE_TO_MULT(wb, 4)) {
    8.33 +        return NULL;
    8.34 +      }
    8.35 +      wb *= 4;
    8.36        break;
    8.37      case MLIB_USHORT:
    8.38      case MLIB_SHORT:
    8.39 -      wb = width * channels * 2;
    8.40 +      if (!SAFE_TO_MULT(wb, 4)) {
    8.41 +        return NULL;
    8.42 +      }
    8.43 +      wb *= 2;
    8.44        break;
    8.45      case MLIB_BYTE:
    8.46 -      wb = width * channels;
    8.47 +      // wb is ready
    8.48        break;
    8.49      case MLIB_BIT:
    8.50 -      wb = (width * channels + 7) / 8;
    8.51 +      if (!SAFE_TO_ADD(7, wb)) {
    8.52 +        return NULL;
    8.53 +      }
    8.54 +      wb = (wb + 7) / 8;
    8.55        break;
    8.56      default:
    8.57        return NULL;
    8.58    }
    8.59  
    8.60 +  if (!SAFE_TO_MULT(wb, height)) {
    8.61 +      return NULL;
    8.62 +  }
    8.63 +
    8.64    data = mlib_malloc(wb * height);
    8.65    if (data == NULL) {
    8.66      return NULL;
     9.1 --- a/src/share/native/sun/awt/medialib/safe_alloc.h	Tue Dec 04 14:02:08 2012 +0000
     9.2 +++ b/src/share/native/sun/awt/medialib/safe_alloc.h	Thu Feb 14 19:51:51 2013 +0400
     9.3 @@ -41,10 +41,4 @@
     9.4      (((w) > 0) && ((h) > 0) && ((sz) > 0) &&                               \
     9.5       (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > ((juint)(sz))))
     9.6  
     9.7 -#define SAFE_TO_MULT(a, b) \
     9.8 -    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
     9.9 -
    9.10 -#define SAFE_TO_ADD(a, b) \
    9.11 -    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
    9.12 -
    9.13  #endif // __SAFE_ALLOC_H__
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/src/share/native/sun/awt/medialib/safe_math.h	Thu Feb 14 19:51:51 2013 +0400
    10.3 @@ -0,0 +1,35 @@
    10.4 +/*
    10.5 + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
    10.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    10.7 + *
    10.8 + * This code is free software; you can redistribute it and/or modify it
    10.9 + * under the terms of the GNU General Public License version 2 only, as
   10.10 + * published by the Free Software Foundation.  Oracle designates this
   10.11 + * particular file as subject to the "Classpath" exception as provided
   10.12 + * by Oracle in the LICENSE file that accompanied this code.
   10.13 + *
   10.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
   10.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   10.16 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   10.17 + * version 2 for more details (a copy is included in the LICENSE file that
   10.18 + * accompanied this code).
   10.19 + *
   10.20 + * You should have received a copy of the GNU General Public License version
   10.21 + * 2 along with this work; if not, write to the Free Software Foundation,
   10.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   10.23 + *
   10.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   10.25 + * or visit www.oracle.com if you need additional information or have any
   10.26 + * questions.
   10.27 + */
   10.28 +
   10.29 +#ifndef __SAFE_MATH_H__
   10.30 +#define __SAFE_MATH_H__
   10.31 +
   10.32 +#define SAFE_TO_MULT(a, b) \
   10.33 +    (((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
   10.34 +
   10.35 +#define SAFE_TO_ADD(a, b) \
   10.36 +    (((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
   10.37 +
   10.38 +#endif // __SAFE_MATH_H__
%d bloggers like this: