• Main Page
  • Related Pages
  • Modules
  • Data Structures
  • Files
  • File List
  • Globals

libavcodec/vmnc.c

Go to the documentation of this file.
00001 /*
00002  * VMware Screen Codec (VMnc) decoder
00003  * Copyright (c) 2006 Konstantin Shishkov
00004  *
00005  * This file is part of FFmpeg.
00006  *
00007  * FFmpeg is free software; you can redistribute it and/or
00008  * modify it under the terms of the GNU Lesser General Public
00009  * License as published by the Free Software Foundation; either
00010  * version 2.1 of the License, or (at your option) any later version.
00011  *
00012  * FFmpeg is distributed in the hope that it will be useful,
00013  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015  * Lesser General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU Lesser General Public
00018  * License along with FFmpeg; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00020  */
00021 
00028 #include <stdio.h>
00029 #include <stdlib.h>
00030 
00031 #include "libavutil/intreadwrite.h"
00032 #include "avcodec.h"
00033 
00034 enum EncTypes {
00035     MAGIC_WMVd = 0x574D5664,
00036     MAGIC_WMVe,
00037     MAGIC_WMVf,
00038     MAGIC_WMVg,
00039     MAGIC_WMVh,
00040     MAGIC_WMVi,
00041     MAGIC_WMVj
00042 };
00043 
00044 enum HexTile_Flags {
00045     HT_RAW =  1, // tile is raw
00046     HT_BKG =  2, // background color is present
00047     HT_FG  =  4, // foreground color is present
00048     HT_SUB =  8, // subrects are present
00049     HT_CLR = 16  // each subrect has own color
00050 };
00051 
00052 /*
00053  * Decoder context
00054  */
00055 typedef struct VmncContext {
00056     AVCodecContext *avctx;
00057     AVFrame pic;
00058 
00059     int bpp;
00060     int bpp2;
00061     int bigendian;
00062     uint8_t pal[768];
00063     int width, height;
00064 
00065     /* cursor data */
00066     int cur_w, cur_h;
00067     int cur_x, cur_y;
00068     int cur_hx, cur_hy;
00069     uint8_t* curbits, *curmask;
00070     uint8_t* screendta;
00071 } VmncContext;
00072 
00073 /* read pixel value from stream */
00074 static av_always_inline int vmnc_get_pixel(const uint8_t* buf, int bpp, int be) {
00075     switch(bpp * 2 + be) {
00076     case 2:
00077     case 3: return *buf;
00078     case 4: return AV_RL16(buf);
00079     case 5: return AV_RB16(buf);
00080     case 8: return AV_RL32(buf);
00081     case 9: return AV_RB32(buf);
00082     default: return 0;
00083     }
00084 }
00085 
00086 static void load_cursor(VmncContext *c, const uint8_t *src)
00087 {
00088     int i, j, p;
00089     const int bpp = c->bpp2;
00090     uint8_t  *dst8  = c->curbits;
00091     uint16_t *dst16 = (uint16_t*)c->curbits;
00092     uint32_t *dst32 = (uint32_t*)c->curbits;
00093 
00094     for(j = 0; j < c->cur_h; j++) {
00095         for(i = 0; i < c->cur_w; i++) {
00096             p = vmnc_get_pixel(src, bpp, c->bigendian);
00097             src += bpp;
00098             if(bpp == 1) *dst8++ = p;
00099             if(bpp == 2) *dst16++ = p;
00100             if(bpp == 4) *dst32++ = p;
00101         }
00102     }
00103     dst8 = c->curmask;
00104     dst16 = (uint16_t*)c->curmask;
00105     dst32 = (uint32_t*)c->curmask;
00106     for(j = 0; j < c->cur_h; j++) {
00107         for(i = 0; i < c->cur_w; i++) {
00108             p = vmnc_get_pixel(src, bpp, c->bigendian);
00109             src += bpp;
00110             if(bpp == 1) *dst8++ = p;
00111             if(bpp == 2) *dst16++ = p;
00112             if(bpp == 4) *dst32++ = p;
00113         }
00114     }
00115 }
00116 
00117 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
00118 {
00119     int i, j;
00120     int w, h, x, y;
00121     w = c->cur_w;
00122     if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x;
00123     h = c->cur_h;
00124     if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y;
00125     x = c->cur_x;
00126     y = c->cur_y;
00127     if(x < 0) {
00128         w += x;
00129         x = 0;
00130     }
00131     if(y < 0) {
00132         h += y;
00133         y = 0;
00134     }
00135 
00136     if((w < 1) || (h < 1)) return;
00137     dst += x * c->bpp2 + y * stride;
00138 
00139     if(c->bpp2 == 1) {
00140         uint8_t* cd = c->curbits, *msk = c->curmask;
00141         for(j = 0; j < h; j++) {
00142             for(i = 0; i < w; i++)
00143                 dst[i] = (dst[i] & cd[i]) ^ msk[i];
00144             msk += c->cur_w;
00145             cd += c->cur_w;
00146             dst += stride;
00147         }
00148     } else if(c->bpp2 == 2) {
00149         uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
00150         uint16_t* dst2;
00151         for(j = 0; j < h; j++) {
00152             dst2 = (uint16_t*)dst;
00153             for(i = 0; i < w; i++)
00154                 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
00155             msk += c->cur_w;
00156             cd += c->cur_w;
00157             dst += stride;
00158         }
00159     } else if(c->bpp2 == 4) {
00160         uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
00161         uint32_t* dst2;
00162         for(j = 0; j < h; j++) {
00163             dst2 = (uint32_t*)dst;
00164             for(i = 0; i < w; i++)
00165                 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
00166             msk += c->cur_w;
00167             cd += c->cur_w;
00168             dst += stride;
00169         }
00170     }
00171 }
00172 
00173 /* fill rectangle with given color */
00174 static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
00175 {
00176     int i, j;
00177     dst += dx * bpp + dy * stride;
00178     if(bpp == 1){
00179         for(j = 0; j < h; j++) {
00180             memset(dst, color, w);
00181             dst += stride;
00182         }
00183     }else if(bpp == 2){
00184         uint16_t* dst2;
00185         for(j = 0; j < h; j++) {
00186             dst2 = (uint16_t*)dst;
00187             for(i = 0; i < w; i++) {
00188                 *dst2++ = color;
00189             }
00190             dst += stride;
00191         }
00192     }else if(bpp == 4){
00193         uint32_t* dst2;
00194         for(j = 0; j < h; j++) {
00195             dst2 = (uint32_t*)dst;
00196             for(i = 0; i < w; i++) {
00197                 dst2[i] = color;
00198             }
00199             dst += stride;
00200         }
00201     }
00202 }
00203 
00204 static av_always_inline void paint_raw(uint8_t *dst, int w, int h, const uint8_t* src, int bpp, int be, int stride)
00205 {
00206     int i, j, p;
00207     for(j = 0; j < h; j++) {
00208         for(i = 0; i < w; i++) {
00209             p = vmnc_get_pixel(src, bpp, be);
00210             src += bpp;
00211             switch(bpp){
00212             case 1:
00213                 dst[i] = p;
00214                 break;
00215             case 2:
00216                 ((uint16_t*)dst)[i] = p;
00217                 break;
00218             case 4:
00219                 ((uint32_t*)dst)[i] = p;
00220                 break;
00221             }
00222         }
00223         dst += stride;
00224     }
00225 }
00226 
00227 static int decode_hextile(VmncContext *c, uint8_t* dst, const uint8_t* src, int ssize, int w, int h, int stride)
00228 {
00229     int i, j, k;
00230     int bg = 0, fg = 0, rects, color, flags, xy, wh;
00231     const int bpp = c->bpp2;
00232     uint8_t *dst2;
00233     int bw = 16, bh = 16;
00234     const uint8_t *ssrc=src;
00235 
00236     for(j = 0; j < h; j += 16) {
00237         dst2 = dst;
00238         bw = 16;
00239         if(j + 16 > h) bh = h - j;
00240         for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
00241             if(src - ssrc >= ssize) {
00242                 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00243                 return -1;
00244             }
00245             if(i + 16 > w) bw = w - i;
00246             flags = *src++;
00247             if(flags & HT_RAW) {
00248                 if(src - ssrc > ssize - bw * bh * bpp) {
00249                     av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00250                     return -1;
00251                 }
00252                 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
00253                 src += bw * bh * bpp;
00254             } else {
00255                 if(flags & HT_BKG) {
00256                     bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00257                 }
00258                 if(flags & HT_FG) {
00259                     fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00260                 }
00261                 rects = 0;
00262                 if(flags & HT_SUB)
00263                     rects = *src++;
00264                 color = !!(flags & HT_CLR);
00265 
00266                 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
00267 
00268                 if(src - ssrc > ssize - rects * (color * bpp + 2)) {
00269                     av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00270                     return -1;
00271                 }
00272                 for(k = 0; k < rects; k++) {
00273                     if(color) {
00274                         fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00275                     }
00276                     xy = *src++;
00277                     wh = *src++;
00278                     paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
00279                 }
00280             }
00281         }
00282         dst += stride * 16;
00283     }
00284     return src - ssrc;
00285 }
00286 
00287 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, const uint8_t *buf, int buf_size)
00288 {
00289     VmncContext * const c = avctx->priv_data;
00290     uint8_t *outptr;
00291     const uint8_t *src = buf;
00292     int dx, dy, w, h, depth, enc, chunks, res, size_left;
00293 
00294     c->pic.reference = 1;
00295     c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
00296     if(avctx->reget_buffer(avctx, &c->pic) < 0){
00297         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
00298         return -1;
00299     }
00300 
00301     c->pic.key_frame = 0;
00302     c->pic.pict_type = FF_P_TYPE;
00303 
00304     //restore screen after cursor
00305     if(c->screendta) {
00306         int i;
00307         w = c->cur_w;
00308         if(c->width < c->cur_x + w) w = c->width - c->cur_x;
00309         h = c->cur_h;
00310         if(c->height < c->cur_y + h) h = c->height - c->cur_y;
00311         dx = c->cur_x;
00312         if(dx < 0) {
00313             w += dx;
00314             dx = 0;
00315         }
00316         dy = c->cur_y;
00317         if(dy < 0) {
00318             h += dy;
00319             dy = 0;
00320         }
00321         if((w > 0) && (h > 0)) {
00322             outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00323             for(i = 0; i < h; i++) {
00324                 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2);
00325                 outptr += c->pic.linesize[0];
00326             }
00327         }
00328     }
00329     src += 2;
00330     chunks = AV_RB16(src); src += 2;
00331     while(chunks--) {
00332         dx = AV_RB16(src); src += 2;
00333         dy = AV_RB16(src); src += 2;
00334         w  = AV_RB16(src); src += 2;
00335         h  = AV_RB16(src); src += 2;
00336         enc = AV_RB32(src); src += 4;
00337         outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00338         size_left = buf_size - (src - buf);
00339         switch(enc) {
00340         case MAGIC_WMVd: // cursor
00341             if(size_left < 2 + w * h * c->bpp2 * 2) {
00342                 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
00343                 return -1;
00344             }
00345             src += 2;
00346             c->cur_w = w;
00347             c->cur_h = h;
00348             c->cur_hx = dx;
00349             c->cur_hy = dy;
00350             if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
00351                 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
00352                 c->cur_hx = c->cur_hy = 0;
00353             }
00354             c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2);
00355             c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2);
00356             c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2);
00357             load_cursor(c, src);
00358             src += w * h * c->bpp2 * 2;
00359             break;
00360         case MAGIC_WMVe: // unknown
00361             src += 2;
00362             break;
00363         case MAGIC_WMVf: // update cursor position
00364             c->cur_x = dx - c->cur_hx;
00365             c->cur_y = dy - c->cur_hy;
00366             break;
00367         case MAGIC_WMVg: // unknown
00368             src += 10;
00369             break;
00370         case MAGIC_WMVh: // unknown
00371             src += 4;
00372             break;
00373         case MAGIC_WMVi: // ServerInitialization struct
00374             c->pic.key_frame = 1;
00375             c->pic.pict_type = FF_I_TYPE;
00376             depth = *src++;
00377             if(depth != c->bpp) {
00378                 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
00379             }
00380             src++;
00381             c->bigendian = *src++;
00382             if(c->bigendian & (~1)) {
00383                 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
00384                 return -1;
00385             }
00386             //skip the rest of pixel format data
00387             src += 13;
00388             break;
00389         case MAGIC_WMVj: // unknown
00390             src += 2;
00391             break;
00392         case 0x00000000: // raw rectangle data
00393             if((dx + w > c->width) || (dy + h > c->height)) {
00394                 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
00395                 return -1;
00396             }
00397             if(size_left < w * h * c->bpp2) {
00398                 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left);
00399                 return -1;
00400             }
00401             paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]);
00402             src += w * h * c->bpp2;
00403             break;
00404         case 0x00000005: // HexTile encoded rectangle
00405             if((dx + w > c->width) || (dy + h > c->height)) {
00406                 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
00407                 return -1;
00408             }
00409             res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]);
00410             if(res < 0)
00411                 return -1;
00412             src += res;
00413             break;
00414         default:
00415             av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
00416             chunks = 0; // leave chunks decoding loop
00417         }
00418     }
00419     if(c->screendta){
00420         int i;
00421         //save screen data before painting cursor
00422         w = c->cur_w;
00423         if(c->width < c->cur_x + w) w = c->width - c->cur_x;
00424         h = c->cur_h;
00425         if(c->height < c->cur_y + h) h = c->height - c->cur_y;
00426         dx = c->cur_x;
00427         if(dx < 0) {
00428             w += dx;
00429             dx = 0;
00430         }
00431         dy = c->cur_y;
00432         if(dy < 0) {
00433             h += dy;
00434             dy = 0;
00435         }
00436         if((w > 0) && (h > 0)) {
00437             outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00438             for(i = 0; i < h; i++) {
00439                 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2);
00440                 outptr += c->pic.linesize[0];
00441             }
00442             outptr = c->pic.data[0];
00443             put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
00444         }
00445     }
00446     *data_size = sizeof(AVFrame);
00447     *(AVFrame*)data = c->pic;
00448 
00449     /* always report that the buffer was completely consumed */
00450     return buf_size;
00451 }
00452 
00453 
00454 
00455 /*
00456  *
00457  * Init VMnc decoder
00458  *
00459  */
00460 static av_cold int decode_init(AVCodecContext *avctx)
00461 {
00462     VmncContext * const c = avctx->priv_data;
00463 
00464     c->avctx = avctx;
00465 
00466     c->pic.data[0] = NULL;
00467     c->width = avctx->width;
00468     c->height = avctx->height;
00469 
00470     if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) {
00471         return 1;
00472     }
00473     c->bpp = avctx->bits_per_coded_sample;
00474     c->bpp2 = c->bpp/8;
00475 
00476     switch(c->bpp){
00477     case 8:
00478         avctx->pix_fmt = PIX_FMT_PAL8;
00479         break;
00480     case 16:
00481         avctx->pix_fmt = PIX_FMT_RGB555;
00482         break;
00483     case 32:
00484         avctx->pix_fmt = PIX_FMT_RGB32;
00485         break;
00486     default:
00487         av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
00488     }
00489 
00490     return 0;
00491 }
00492 
00493 
00494 
00495 /*
00496  *
00497  * Uninit VMnc decoder
00498  *
00499  */
00500 static av_cold int decode_end(AVCodecContext *avctx)
00501 {
00502     VmncContext * const c = avctx->priv_data;
00503 
00504     if (c->pic.data[0])
00505         avctx->release_buffer(avctx, &c->pic);
00506 
00507     av_free(c->curbits);
00508     av_free(c->curmask);
00509     av_free(c->screendta);
00510     return 0;
00511 }
00512 
00513 AVCodec vmnc_decoder = {
00514     "vmnc",
00515     CODEC_TYPE_VIDEO,
00516     CODEC_ID_VMNC,
00517     sizeof(VmncContext),
00518     decode_init,
00519     NULL,
00520     decode_end,
00521     decode_frame,
00522     .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),
00523 };
00524 

Generated on Sat Feb 16 2013 09:23:14 for ffmpeg by  doxygen 1.7.1