*** ../diablo/util/diablo.c	Sun Nov 23 20:32:29 1997
--- util/diablo.c	Mon Dec 15 20:32:27 1997
***************
*** 22,27 ****
--- 22,32 ----
   * (c)Copyright 1997, Matthew Dillon, All Rights Reserved.  Refer to
   *    the COPYRIGHT file in the base directory of this distribution 
   *    for specific rights granted.
+  *
+  * Modified 12/4/1997 to include support for compressed data streams.
+  * Modifications (c) 1997, Christopher M Sedore, All Rights Reserved.
+  * Modifications may be distributed freely as long as this copyright
+  * notice is included without modification.
   */
  
  #include "defs.h"
***************
*** 1221,1227 ****
      Buffer *bi;
      FILE *fo;
      char *buf;
!     int streamMode = 0;
      int syntax = 0;
      int unimp = 0;
  
--- 1226,1232 ----
      Buffer *bi;
      FILE *fo;
      char *buf;
!     int streamMode = 0, compressMode=0;
      int syntax = 0;
      int unimp = 0;
  
***************
*** 1623,1628 ****
--- 1628,1639 ----
  		xfprintf(fo, "203 StreamOK.\r\n");
  	    } else if (p && strcasecmp(p, "reader") == 0) {
  		unimp = 1;
+ 	    } else if (p && strcasecmp(p, "compress") == 0) { /* added for compression support */
+ 		bi->bu_CBuf=malloc(sizeof(CompressBuffer));
+                 bzero(bi->bu_CBuf,sizeof(CompressBuffer));
+                 inflateInit(&bi->bu_CBuf->z_str);
+                 compressMode=1;
+ 		xfprintf(fo, "207 Compression enabled\r\n");
  	    } else {
  		syntax = 1;
  	    }
***************
*** 1713,1718 ****
--- 1724,1734 ----
  	    sprintf(ArgvBuf, "%-*.*s", ArgvLen, ArgvLen, tbuf);
  	}
      }
+ 
+     if (bi->bu_CBuf) {
+       syslog(LOG_INFO,"%-20s compbytes=%.0f decompbytes=%.0f (%.2f%% compression)",HName,bi->bu_CBuf->orig,bi->bu_CBuf->decomp,100 - ((bi->bu_CBuf->orig/bi->bu_CBuf->decomp)*100));
+     }
+ 
      bclose(bi, 1);
      fclose(fo);
  }
*** ../diablo/util/dnewslink.c	Fri Nov  7 18:36:35 1997
--- util/dnewslink.c	Thu Jan  1 16:07:46 1998
***************
*** 36,41 ****
--- 36,46 ----
   * (c)Copyright 1997, Matthew Dillon, All Rights Reserved.  Refer to
   *    the COPYRIGHT file in the base directory of this distribution 
   *    for specific rights granted.
+  *
+  * Modified 12/4/1997 to include support for compressed data streams.
+  * Modifications (c) 1997, Christopher M Sedore, All Rights Reserved.
+  * Modifications may be distributed freely as long as this copyright
+  * notice is included without modification.
   */
  
  #include "defs.h"
***************
*** 76,81 ****
--- 81,87 ----
  #define OK_SLAVE        202     /* Slave status noted */
  #define OK_STREAMOK	203	/* Can-do streaming	*/
  #define OK_GOODBYE      205     /* Closing connection */
+ #define OK_COMPRESSOK	207	/* Can-do compression */
  #define OK_GROUP        211     /* Group selected */
  #define OK_GROUPS       215     /* Newsgroups follow */
  #define OK_ARTICLE      220     /* Article (head & body) follows */
***************
*** 202,207 ****
--- 208,230 ----
  int TryStreaming = 1;
  int BytesPend = 0;
  
+ /*
+  * added to support compression, cmsedore@maxwell.syr.edu 12/4/97
+  */
+ 
+ #define COMPRESS_OFF 0
+ #define COMPRESS_ON  1
+ 
+ int CompressMode = 0;
+ int TryCompress = 1;
+ int32 CompressedBytes=0;
+ int32 RawBytes=0;
+ int32 CompressedBytesTotal=0;
+ int32 RawBytesTotal=0;
+ z_stream z_str;
+ char CompressBuf[8192]; 
+ int CompressNextFlush=0;
+ 
  int LogAfterCount = -1;
  int CloseReopenCount = -1;
  int KillFd = -1;
***************
*** 332,337 ****
--- 355,363 ----
  	case 'c':
  	    CloseReopenAfter = strtol(((*ptr) ? ptr : av[++i]), NULL, 0);
  	    break;
+         case 'C':
+             TryCompress=0;
+             break;
  	}
      }
      LogAfterCount += LogAfter;
***************
*** 382,387 ****
--- 408,418 ----
  	AcceptedBytes = 0;
  	AcceptedBytesTotal = 0;
  
+ 	CompressedBytes = 0;
+ 	RawBytes = 0;
+ 	CompressedBytesTotal = 0;
+ 	RawBytesTotal = 0;
+ 
  	AcceptedTotal = 0;
  	RefusedTotal = 0;
  	RejectedTotal = 0;
***************
*** 780,786 ****
  	    TheirMs += TheirMsTotal;
  	    MsCount += MsCountTotal;
  
! 	    logStats("final");
  
  	    /*
  	     * be nice
--- 811,818 ----
  	    TheirMs += TheirMsTotal;
  	    MsCount += MsCountTotal;
  
!             RawBytes += RawBytesTotal;
!             CompressedBytes += CompressedBytesTotal;
  
  	    /*
  	     * be nice
***************
*** 794,800 ****
--- 826,838 ----
  		KillFd = -1;
  		/* normal quit, do not set cfd to -1 	     */
  		/* this allows us to go on to the next batch */
+ 
+ 		if (CompressMode == COMPRESS_ON)
+ 			deflateEnd(&z_str);
  	    }
+ 
+ 	    logStats("final");
+ 
  	} /* cfd=connectTo */
  
  	if (fd > 0) {
***************
*** 818,823 ****
--- 856,866 ----
  
      clearResponseBuf();
  
+     if (CompressMode == COMPRESS_ON)
+ 	deflateEnd(&z_str);
+ 
+     CompressMode=COMPRESS_OFF;
+ 
      memset(&sin, 0, sizeof(sin));
      ++ConnectCount;
  
***************
*** 963,987 ****
  
      if (cfd >= 0) {
  	char *ptr;
  
  	if (TryStreaming) {
  	    switch(commandResponse(cfd, &ptr, "mode stream\r\n")) {
  	    case OK_STREAMOK:
  		StreamMode = STREAM_ON;
! 		logit("connect: %s (streaming)\n", ptr);
! 		bsprintf("connect: stream");
  		break;
  	    default:
  		StreamMode = STREAM_OFF;
! 		logit("connect: %s (nostreaming)\n", ptr);
! 		bsprintf("connect: nostream");
  		break;
  	    }
  	} else {
! 	    logit("connect (streaming disabled)\n");
! 	    bsprintf("connect (streaming disabled)\n");
  	    StreamMode = STREAM_OFF;
  	}
      }
  
      /*
--- 1006,1051 ----
  
      if (cfd >= 0) {
  	char *ptr;
+         char negotiated[255];
+ 
+         negotiated[0]=0;
  
  	if (TryStreaming) {
  	    switch(commandResponse(cfd, &ptr, "mode stream\r\n")) {
  	    case OK_STREAMOK:
  		StreamMode = STREAM_ON;
!                 strcat(negotiated,"streaming enabled");                
  		break;
  	    default:
  		StreamMode = STREAM_OFF;
!                 strcat(negotiated,"streaming disabled");                
  		break;
  	    }
  	} else {
!             strcat(negotiated,"streaming disabled");                
  	    StreamMode = STREAM_OFF;
  	}
+         /* added to support compression, cmsedore@maxwell.syr.edu 12/4/97 */
+ 	if (TryCompress) {
+ 	    switch(commandResponse(cfd, &ptr, "mode compress\r\n")) {
+ 	    case OK_COMPRESSOK:
+ 		CompressMode = COMPRESS_ON;
+                 strcat(negotiated," compression enabled");                
+                 bzero(&z_str,sizeof(z_stream));
+                 deflateInit(&z_str,Z_DEFAULT_COMPRESSION);
+ 		break;
+ 	    default:
+                 strcat(negotiated," compression disabled");                
+ 		CompressMode = COMPRESS_OFF;
+ 		break;
+ 	    }
+ 	} else {
+             strcat(negotiated," compression disabled");                
+ 	    CompressMode = COMPRESS_OFF;
+ 	}
+ 
+ 	logit("connect: %s (%s)\n", ptr, negotiated);
+ 	bsprintf("connect: %s",negotiated);
      }
  
      /*
***************
*** 1097,1102 ****
--- 1161,1169 ----
      int ccfd = dup(cfd);
      int multiArtFile = 0;
  
+     char buf[9000];
+     int bcount=0;
+ 
      sprintf(path, "%s/%s", SPOOL, relPath);
  
      if (cfd >= 0 &&
***************
*** 1125,1143 ****
  	     * if first character is a '.', escape it
  	     */
  
! 	    if (ptr[i] == '.')
! 		fputc('.', fo);
  
  	    /*
  	     * dump the article.
  	     */
  
! 	    while (i < size && ptr[i] != '\n')
! 		++i;
! 	    fwrite(ptr + b, i - b, 1, fo);
! 	    fputc('\r', fo);
! 	    fputc('\n', fo);
! 	    ++i;
  
  	    /*
  	     * if i > fsize, we hit the end of the file without
--- 1192,1241 ----
  	     * if first character is a '.', escape it
  	     */
  
!             if (ptr[i] == '.') {
!   	      if (!CompressMode) {
! 	  	fputc('.', fo);
!                } else {        
!                  /* added to support compression, cmsedore@maxwell.syr.edu 12/4/97 */
!                   buf[bcount]='.'; bcount++;
!                }
!             }
  
  	    /*
  	     * dump the article.
  	     */
  
!             if (!CompressMode) {
!   	        while (i < size && ptr[i] != '\n')
! 		  ++i;
!   	    	fwrite(ptr + b, i - b, 1, fo);
! 	    	fputc('\r', fo);
! 	    	fputc('\n', fo);
! 	    	++i;
!             } else {
!                  /* added to support compression, cmsedore@maxwell.syr.edu 12/4/97 */
!                 
!                 while ((i<size) && (ptr[i]!='\n')) {
!                   buf[bcount]=ptr[i]; bcount++; i++;
!                    if (bcount>8000) {
!                       z_str.next_in=buf;
!                       z_str.avail_in=bcount;
!                       RawBytes+=bcount;
!                       while (z_str.avail_in) {
!                         z_str.next_out=CompressBuf;
!                         z_str.avail_out=8192;
!                         deflate(&z_str,Z_SYNC_FLUSH);
!                         fwrite(CompressBuf,8192-z_str.avail_out,1,fo);
!                         CompressedBytes+=(8192-z_str.avail_out);
!                       }
!                       bcount=0;
!                    }
!                 }   
!                 buf[bcount]='\r'; bcount++;
!                 buf[bcount]='\n'; bcount++;
!                 ++i;
!              }
!                
  
  	    /*
  	     * if i > fsize, we hit the end of the file without
***************
*** 1154,1159 ****
--- 1252,1276 ----
      if (ptr)
  	cdunmap(ptr, size, multiArtFile);
  
+     /*
+      * Flush any data existing in our buffer for data to-be-compressed.
+      * Added 12/7/97 cmsedore@maxwell.syr.edu to support compression.
+      */
+ 
+     if (bcount) {
+       z_str.next_in=buf;
+       z_str.avail_in=bcount;
+       RawBytes+=bcount;
+       while (z_str.avail_in) {
+         z_str.next_out=CompressBuf;
+         z_str.avail_out=8192;
+         deflate(&z_str,Z_NO_FLUSH);
+         fwrite(CompressBuf,8192-z_str.avail_out,1,fo);
+         CompressedBytes+=(8192-z_str.avail_out);
+       }
+       bcount=0;
+     }
+ 
      if (fo)
  	fclose(fo);
      else if (ccfd >= 0)
***************
*** 1496,1501 ****
--- 1613,1626 ----
  #endif
  	LastErrBuf
      );
+   
+     if (CompressMode)
+ 	logit("%s compbytes=%ld decompbytes=%ld (%.2f%% compression)\n",
+ 	    description,
+ 	    CompressedBytes,
+ 	    RawBytes,
+ 	    100 - (((float)CompressedBytes / (float)RawBytes) * 100)
+ 	);
      DeltaStart = time(NULL);
      AcceptedCnt = 0;
      AcceptedBytes = 0;
***************
*** 1503,1508 ****
--- 1628,1638 ----
      RejectedCnt = 0;
      Count = 0;
  
+     RawBytesTotal+=RawBytes;
+     CompressedBytesTotal+=CompressedBytes;
+     RawBytes=0;
+     CompressedBytes=0;
+ 
      MsCountTotal += MsCount;
      OurMsTotal += OurMs;
      TheirMsTotal += TheirMs;
***************
*** 1540,1546 ****
  	vsprintf(tmp, ctl, va);
  	va_end(va);
  	credtime(OUR_DELAY);
! 	write(cfd, tmp, strlen(tmp));
  	credtime(THEIR_DELAY);
  
  	if (DebugOpt > 1) {
--- 1670,1689 ----
  	vsprintf(tmp, ctl, va);
  	va_end(va);
  	credtime(OUR_DELAY);
!         if (!CompressMode) {
!   	  write(cfd, tmp, strlen(tmp));
!         } else {
!           z_str.next_in=tmp;
!           z_str.avail_in=strlen(tmp);
!           RawBytes+=z_str.avail_in;
!           while (z_str.avail_in) {
!             z_str.next_out=CompressBuf;
!             z_str.avail_out=8192;
!             deflate(&z_str,Z_SYNC_FLUSH);
!             write(cfd,CompressBuf,8192-z_str.avail_out);
!             CompressedBytes+=(8192-z_str.avail_out);
!           }
!         }
  	credtime(THEIR_DELAY);
  
  	if (DebugOpt > 1) {
*** ../diablo/util/dspoolout.c	Fri Nov  7 18:37:14 1997
--- util/dspoolout.c	Mon Dec 15 18:12:41 1997
***************
*** 38,43 ****
--- 38,44 ----
  #define SF_NOSTREAM	0x0001
  #define SF_REALTIME	0x0002
  #define SF_NOBATCH	0x0004
+ #define SF_COMPRESS	0x0008
  
  int MaxRun = 2;		/* default MaxRun				       */
  int MinFlushSecs = 0;	/* minimum time between flushes if queue not caught up */
***************
*** 185,190 ****
--- 186,193 ----
  				flags |= SF_REALTIME;
  			    } else if (strcmp(maxqStr, "nobatch") == 0) {
  				flags |= SF_NOBATCH;
+ 			    } else if (strcmp(maxqStr, "compress") == 0) {
+ 				flags |= SF_COMPRESS;
  			    } else if (strncmp(maxqStr, "bind=", 5) == 0) {
  				obip = maxqStr + 5;
  			    } else if (maxqStr[0] == 'q') {
***************
*** 484,489 ****
--- 487,493 ----
  				"-P", portBuf,
  				"-D",
  				((flags & SF_NOSTREAM) ? "-i" : "-nop"),
+ 				((flags & SF_COMPRESS) ? "-nop" : "-C"),
  				outboundIpStr,
  				txBufSizeStr,
  				rxBufSizeStr,
***************
*** 553,558 ****
--- 557,563 ----
  				"-N", "100",	/* 1000 spool cycles	*/
  				((delay < 0) ? "-r-1" : "-r"),
  				((flags & SF_NOSTREAM) ? "-i" : "-nop"),
+ 				((flags & SF_COMPRESS) ? "-nop" : "-C"),
  				outboundIpStr,
  				txBufSizeStr,
  				rxBufSizeStr,
*** ../diablo/lib/buffer.c	Sat Sep 27 02:26:44 1997
--- lib/buffer.c	Wed Dec 17 13:15:33 1997
***************
*** 7,12 ****
--- 7,18 ----
   *    for specific rights granted.
   *
   * Buffered I/O, may be used for reading or writing, but not both.
+  *
+  * Modified 12/4/1997 to include support for compressed data streams.
+  * Modifications (c) 1997, Christopher M Sedore, All Rights Reserved.
+  * Modifications may be distributed freely as long as this copyright 
+  * notice is included without modification.
+  *
   */
  
  #include "defs.h"
***************
*** 157,165 ****
  	 * otherwise read some data and repeat
  	 */
  	{
! 	    int n;
  
! 	    n = read(b->bu_Fd, b->bu_Data + b->bu_End, b->bu_DataMax - b->bu_End);
  
  	    /*
  	     * EOF
--- 163,272 ----
  	 * otherwise read some data and repeat
  	 */
  	{
! 	    int n=0;
! 
!             if (b->bu_CBuf) {
!               /*
!                * The block contains the mods to deal with compressed
!                * streams. cmsedore@maxwell.syr.edu 12/4/97
!                */
!               int nc,rc,code,decompcount;
! 
!               if (b->bu_CBuf->dataError)
!                 return ((void *) -1);
!               
!               while (!n) {
!                 
!                 /* if we have bytes left over, process them before reading */
! 
!                 if (b->bu_CBuf->z_str.avail_out!=-1)  {
!                   rc=COMPRESS_BUFFER_LENGTH-b->bu_CBuf->z_str.avail_in;
!                   /* check for decompressor buffer overflow */
!                   if (rc==0) {
! 		syslog(LOG_ERR, "decompressor buffer overflow fd %d",b->bu_Fd);
!    		    return((void *)-1);
!                   }
!                   nc = read(b->bu_Fd,&b->bu_CBuf->cb_Buf[b->bu_CBuf->z_str.avail_in],rc);
!                   if (nc < 1) { 
!                     syslog(LOG_ERR,"eof (%i) on read",nc);
!                     n=nc; goto err_or_eof;
!                   }   
!                   b->bu_CBuf->orig=b->bu_CBuf->orig + (double) nc;
!                   b->bu_CBuf->z_str.next_in=b->bu_CBuf->cb_Buf; 
!                   b->bu_CBuf->z_str.avail_in=rc=nc+b->bu_CBuf->z_str.avail_in;
!                 } else nc=0;
!              
!                  b->bu_CBuf->z_str.avail_out=decompcount=b->bu_DataMax-b->bu_End;
!                  b->bu_CBuf->z_str.next_out=b->bu_Data+b->bu_End;
!               
!                  code=inflate(&b->bu_CBuf->z_str,Z_PARTIAL_FLUSH);
! 
! 
!                  /*
!                   * Z_BUF_ERROR indicates that no progress was possible due
!                   * to buffering constraints.  We try again, this time
!                   * we'll do a read() for sure.  We may not have before
!                   * because z_str.avail_out==0
!                   */
! 
!                  if (code==Z_BUF_ERROR) {
! #ifdef COMPDEBUG 
! syslog(LOG_ERR,"code=%i,read=%u,decomp=%u,avail_in=%u,avail_out=%u",code,nc,decompcount,b->bu_CBuf->z_str.avail_in,b->bu_CBuf->z_str.avail_out);
! #endif               
!                    continue;
!                  }
! 
!                  /* 
!                   * this indicates an inconsistency in the z_str structure
!                   * members.  We don't mess with them.  I *think* this 
!                   * happens when we hit z_str.avail_in=0 and
!                   * z_str.avail_out=0 simultaneously.  
!                   *
!                   * at this point, we continue.
!                   */
!          
!                  if (code==Z_STREAM_ERROR) {
! syslog(LOG_ERR,"code=%i,read=%u,decomp=%u,avail_in=%u,avail_out=%u",code,nc,decompcount,b->bu_CBuf->z_str.avail_in,b->bu_CBuf->z_str.avail_out);
!                    continue;
!                  }                 
! 
!                  /*
!                   * any other error probably means we're done.  We set the
!                   * dataError member so that when diablo reads again, we
!                   * indicate that this stream is exhausted.
!                   *
!                   * future improvements might try recovery.
!                   */
! 
!                  if (code!=Z_OK) {
! 		syslog(LOG_ERR, "inflate error (%i) on fd %d: %s",code,b->bu_Fd,b->bu_CBuf->z_str.msg);
!                     b->bu_CBuf->dataError=1;
!                    return((void *)-1);
!                  }
! 
!                  decompcount-=b->bu_CBuf->z_str.avail_out; 
! 
!                  if (b->bu_CBuf->z_str.avail_out==0)
!                    b->bu_CBuf->z_str.avail_out=-1;
!                  else 
!                    if (b->bu_CBuf->z_str.avail_in) {
!                      if ((char *)b->bu_CBuf->z_str.next_in!=(char *)b->bu_CBuf->cb_Buf) {
!                        memmove(b->bu_CBuf->cb_Buf,b->bu_CBuf->z_str.next_in,b->bu_CBuf->z_str.avail_in);
!                        b->bu_CBuf->z_str.next_in=b->bu_CBuf->cb_Buf;
!                      }
!                    }
! 
!                  n=decompcount;
!                  b->bu_CBuf->decomp=b->bu_CBuf->decomp + (double) decompcount;
! 
!               } /* while (!n) */
!                  
!             } else {               
!   	      n = read(b->bu_Fd, b->bu_Data + b->bu_End, b->bu_DataMax - b->bu_End);
!             }
! 
  
! err_or_eof:
  
  	    /*
  	     * EOF
*** ../diablo/lib/defs.h	Sat Nov 22 04:21:09 1997
--- lib/defs.h	Mon Dec 15 15:52:26 1997
***************
*** 7,12 ****
--- 7,18 ----
   * (c)Copyright 1997, Matthew Dillon, All Rights Reserved.  Refer to
   *    the COPYRIGHT file in the base directory of this distribution
   *    for specific rights granted.
+  *
+  * Modified 12/4/1997 to include support for compressed data streams.
+  * Modifications (c) 1997, Christopher M Sedore, All Rights Reserved.
+  * Modifications may be distributed freely as long as this copyright
+  * notice is included without modification.
+  *
   */
  
  #include "config.h"
***************
*** 290,295 ****
--- 296,318 ----
  
  #define MAXFEEDTABLE	256
  
+ /*
+  * Additions to support compressed streams
+  * cmsedore@maxwell.syr.edu 12/4/97
+  */
+ 
+ #include "zlib.h"
+ 
+ #define COMPRESS_BUFFER_LENGTH 8192
+ 
+ typedef struct compressBuffer {
+   char			cb_Buf[COMPRESS_BUFFER_LENGTH];
+   int			cb_compressHoldoverCount;
+   int			dataError;
+   double		orig,decomp;
+   z_stream		z_str;
+ } CompressBuffer;
+ 
  typedef struct Buffer {
      int			bu_Beg;
      int			bu_NLScan;
***************
*** 298,306 ****
--- 321,331 ----
      int			bu_DataMax;	/* extended maximum	*/
      int			bu_Fd;
      int			bu_Error;
+     CompressBuffer	*bu_CBuf;
      char		*bu_Data;	/* included/extended buffer	*/
      char		bu_Buf[1024];	/* included buffer		*/
  } Buffer;
+ 
  
  #include <lib/mem.h>
  #include <lib/kpdb.h>
*** ../diablo/XMakefile.inc	Fri Sep 26 01:49:33 1997
--- XMakefile.inc	Tue Dec  9 16:55:13 1997
***************
*** 16,22 ****
  .set CC		cc
  #endif
  
! .set LFLAGS	-Lobj -ldiablo -lm
  .set IDIR	/news
  .set MDIR	/usr/local/man
  
--- 16,22 ----
  .set CC		cc
  #endif
  
! .set LFLAGS	-Lobj -ldiablo -lm -lz
  .set IDIR	/news
  .set MDIR	/usr/local/man
  
