/*
 * $Header: /home/cvs/webget/c/http,v 1.38 2001/03/11 17:08:04 joseph Exp $
 *
 */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <time.h>

#include "wimplib.h"

#include "swis.h"

#include "netdb.h"
#include "sys/socket.h"
#include "sys/errno.h"

#include "macros.h"
#include "fetchstruc.h"
#include "my_string.h"
#include "proxy.h"
#include "fdset.h"
#include "socket.h"
#include "fetch.h"
#include "eval.h"
#include "file.h"
#include "version.h"
#include "debug.h"
#include "misc.h"
#include "config.h"
#include "flex.h"
#include "status.h"

#include "http.h"



static FILE *http_logf = NULL; /* FetchLog file */
static FILE *http_errlogf = NULL; /* FetchError file */

typedef enum { error_hard, error_soft, error_forgiveable } http_errortype;


int http_bytesrxed; /* updated in here */
int http_waiting; /* updated in url.c */
int http_fetched; /* update in url.c */
int http_fetching; /* update in fetch.c, (mainly?) */

void http_zerocounts( void )
{
  http_bytesrxed = 0;
  http_waiting = 0;
  http_fetched = 0;
  http_fetching = 0;
  status_update();
}

static void http_error( int no, http_errortype type, const char *error );


/* http_addbasehref()
 *
 * NB. Call with care as will probably cause all flex pointers to budge...
 */
static int http_addbasehref( int no, const char *base )
{
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  int urllen = strlen( url ) + 1;
  int baselen = strlen( base ) + 1;

  /* calc size needed */
  int wewant = sizeof(httpfetch_t) + urllen + baselen;

  if ( !flex_extend( (flex_ptr) &httpfetch[no], wewant ) )
  {
    debug_printf(("Flex_extend failed in http_addbasehref!\n"));
    return -1;
  }
  url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  url += urllen;

  memcpy( url, base, baselen );
  httpfetch[no]->baseurl = 1;
  debug_printf(("Set base url to '%s'\n",base));

  return 0;
}
// #define my_strcat(a,b) { strncat(a,b, sizeof(a) - (strlen(a)+1) ); } */

/* http_abortoutput()
 *
 * Close output file if open, then delete if
 *
 */
static int http_abortoutput(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  char name[FILEPATH_MAXLEN];
  int ret;

//  if (!x->html) return 0;

  if ( x->html )
  {
    fclose(x->html);
    x->html=NULL;
  }

  ret = misc_urltofilename( url, name, sizeof name );
  if (ret<0) return ret;

  remove(name);
  return 0;
}

void http_abort( int no )
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  http_abortoutput(no);
  fdset_close(&x->sock);
  httpfetch_removepoll(no);
  x->idle = 1;
}


static int http_closeoutput(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  char name[FILEPATH_MAXLEN];
  int ret;

  if (!x->html) return 0;

  fclose(x->html);
  x->html=NULL;

  ret = misc_urltofilename( url, name, sizeof name );
  if (ret<0) return ret;

  /* Set the filetype */
  if ( x->filetype != -1 )
  {
    file_settype(name, x->filetype);
  }
  if ( x->date != 0 )
  {
    file_setstampunix( name, x->date );
  }

  return 0;
}


static void http_fetchfinished(int no, const char *note)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int ret, success;
  char tmp[10];
  FILE *f;
  char *url;
  /* there SHOULDN'T be any more data on the socket...
   * so what happens when we try and read from it?
   */

  debug_printf(("%d : finished\n",no));

  if (x->respcode==301) debug_printf(("%d : output closed\n",no));

  ret = recv(x->sock, tmp, sizeof(tmp), MSG_PEEK); /* don't actually take any data */
  if (x->respcode==301) debug_printf(("%d : recv = %d\n",no,ret));

  if ( ret > 0 || x->bufflen )
  {
    success=0;
    debug_printf(("%d : Data waiting on socket after fetch finished!(bufflen=%d)\n",no,x->bufflen));
    fdset_close(&x->sock);
    x->bufflen = 0;
    http_error( no, error_soft, "Server sent wrong size" );
    return;
  }

  if ( ! (ret == -1 && errno == EWOULDBLOCK) )
    fdset_close( &x->sock ); /* close socket on any error other than ewouldblock */

  url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  success = 0;

  if (x->respcode==200 || x->respcode==304)
    success = 1;

  if (x->respcode==301 || x->respcode==302 || x->respcode==404 || x->respcode==500)
    success = 2; /* Don't log, we've already logged */

  x->idle = 1; /* this session no longer in use! */
  http_closeoutput( no );

  f = NULL;
  if ( success == 1 )
    f = http_logf;
  else if ( success == 2 )
    { /* already logged when Location: header was received */ }
  else
    f = http_errlogf;

  if (f)
  {
    char datestring[80];
    time_t date = time(NULL);
    strftime(datestring, sizeof(datestring), "%a %b %d %H:%M:%S", localtime(&date));
    if (success==1)
    {
      if ( note )
        fprintf(f,"%s %s %3d %s\n", datestring, url, x->respcode, note);
      else
        fprintf(f,"%s %s %3d\n", datestring, url, x->respcode);
    }
    else
    {
      if ( note )
        fprintf(f,"%s %s http error code %3d %s\n", datestring, url, x->respcode, note);
      else
        fprintf(f,"%s %s http error code %3d\n", datestring, url, x->respcode);
    }
  }

  /* move url to fetched list */
  url_addto( FETCHED, url, x->linkdepthtogo, x->fetchportion, x->noinlineimages, x->inlineframes, x->ruleset,
             inline_not /* doesn't really matter */, x->act_linkdepth, x->imagefetchportion );
  /* url invalid now */

  if ( httpfetch_kick(no) <= 0 )
    status_seturl( no, "" );
}

static void http_adderrorlog( const char *url, const char *error )
{
  FILE *f = http_errlogf;

  if (f)
  {
    char datestring[80];
    time_t date = time( NULL );
    strftime( datestring, sizeof(datestring), "%a %b %d %H:%M:%S", localtime(&date) );
    fprintf( f, "%s %s %s\n", datestring, url, error);
  }
  else
  {
    debug_printf(("xx :Could not write to fetcherror\n"));
  }
}



static void http_error( int no, http_errortype type, const char *error )
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int retry = x->retries > 0 && (type == error_soft || type== error_forgiveable);

  debug_printf(("%d : !!!! %s error (%s) : ",no,type==error_soft?"Soft":(type==error_hard?"Hard":"Forgiveable"),error));

  /* Stop what we're doing */
  if ( retry || type != error_forgiveable )
  {
    http_abortoutput(no);
    fdset_close(&x->sock);
    httpfetch_removepoll(no);
  }

  /* If we got some 'soft' error (socket error or other, indicating a failure somewhere), we go back,
  resolve again and try connecting. */
  if ( retry )
  {
    debug_printf(("Retrying!!!!\n"));
    http_init(no,1);
    return;
  }

  if ( type == error_forgiveable )
  {
    http_fetchfinished( no, error );
    return;
  }

  debug_printf(("Gave up!\n"));
  {
    char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
    http_adderrorlog( url, error );
  }

  x->idle = 1;

  /* move url to fetched list */
  {
    char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
    url_addto( FETCHED, url, x->linkdepthtogo, x->fetchportion, x->noinlineimages, x->inlineframes, x->ruleset,
               inline_not /* doesn't really matter */, x->act_linkdepth, x->imagefetchportion );
    /* url invalid, x invalid */
  }

  httpfetch_kick(no);
  return;
}


/* takes a url, copies the hostname into 'host' and returns a ptr to the start of the path section */
static char *http_spliturl(const char *url, char *host)
{
  static char rootpath[]="/"; /* needed for silly url's without trailing /'s... */
  char *ptr = strstr(url,"//");
  char *end;

  *host=0;
  if (ptr==NULL) return NULL;
  ptr+=2;
  end = strchr(ptr,'/');
  if (end==NULL)
  {
    /* the URL is missing the trailing / */
    /* copy host, return a / as path */
    strncat(host, ptr, HOST_MAXLEN-1);
    return rootpath;
  }
  strncat(host, ptr, MIN(HOST_MAXLEN-1, end-ptr));
  return end;
}

/* x->proxy values : 0 -> not yet determined, 1 -> no proxy, 2 -> proxy[0], 3 -> proxy[1], etc */
static void http_setproxy(int no)
{
  char host[HOST_MAXLEN];
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  int ret;

  http_spliturl( url, host);

  ret = proxy_usefor( url, host );
//  debug_printf(("proxy_usefor(%s,%s)=%d\n",url,host,ret));
  if ( ret == 0 )
  {
    x->proxy = 1; /* no proxy */
    return;
  }
  x->proxy = ret + 1; /* 1(http) -> 2, 2(ftp) -> 3, etc */
}


static int http_requestsend(int no)
{
  char filename[ FILEPATH_MAXLEN ];
  char date[30];
  char stamp[ 5 ];
  int doif;
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char request[REQUEST_MAXLEN];
  char *type = ( x->where == PENDING ) ? "GET " : "HEAD ";
  int ret,size;
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  char host[HOST_MAXLEN], *path; /* for http_spliturl */

  if (!x->proxy) return -1; /* proxy not initialised. :-( */

  /* build up the request into 'request' */
  strcpy(request,type); /* can't overflow */

  path = http_spliturl( url, host);

  /* see if we've already got the file... do an if-modified-since if we have */
  doif = 1;
  ret = misc_urltofilename( url, filename, sizeof filename );
  if ( ret >= 0 ) ret = file_readstamp( filename, stamp );
  if ( ret >= 0 )
  {
    struct
    {
      int csec;
      int sec;
      int min;
      int hour;
      int day;
      int month;
      int year;
      int dayofweek;
      int dayofyear;
    } ord;
    const char *month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
                            "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
    const char *day[] = { "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" };

    ret = _swix( Territory_ConvertTimeToUTCOrdinals, _INR(1,2), stamp, &ord ) ? -1 : 0;

    if ( ret >= 0 )
    {
      /* We're aiming for a format like : Mon, 03 Feb 1998 20:09:59 GMT */
      sprintf( date, "%s, %02d %s %04d %02d:%02d:%02d GMT",
                     day[ ord.dayofweek ], ord.day, month[ ord.month ], ord.year,
                     ord.hour, ord.min, ord.sec );
    }
  }
  if ( ret < 0 ) doif = 0;

  if (x->proxy==1)
  {
    /* no proxy. need to send host: header, but only url path */
    strncat(request, path,                     sizeof(request) - (strlen(request)+1));
  }
  else
  {
    /* proxy. no host header, send full url. */
    strncat(request, url,                      sizeof(request) - (strlen(request)+1));
  }
  strncat(request, " " HTTP_VER "\r\nHost:", sizeof(request) - (strlen(request)+1));
  strncat(request, host,                     sizeof(request) - (strlen(request)+1));
  /* User-Agent's seem to be like:
  User-Agent: Mozilla/2.02 (compatible; ANT Argo Fresco/1.70; RISC OS 3.70)
  User-Agent: Mozilla/4.01 (Compatible; Acorn Browse 1.33 [12-Feb-98] Beta; RISC OS 3.70) Acorn-HTTP/0.75
  */

  strncat( request, "\r\nUser-Agent: Mozilla/4.01 (Compatible; "TASK_NAME" "VERSION")",
           sizeof(request) - (strlen(request)+1) );
  if ( 0 /*doif*/)
  {
    strncat(request, "\r\nIf-Modified-Since:", sizeof(request) - (strlen(request)+1));
    strncat(request, date,                     sizeof(request) - (strlen(request)+1));
    debug_printf(("%d : If-Modified-Since: %s\n",no,date));
  }
  strncat(request, "\r\nAccept: text/html, image/gif, image/jpeg, image/pjpeg, image/png, */*; q=.2",
                   sizeof(request) - (strlen(request)+1));
  strncat(request, "\r\n\r\n",                 sizeof(request) - (strlen(request)+1));

  size = strlen(request);

  if ( (size+1) == sizeof(request) ) return -1; /* probably truncated request. permanent failure */

  debug_printf(("%d : Sending request:'%s'\n",no,url));


  ret = send(x->sock, request, size, 0);
  if (ret==-1)
  {
    if (errno==EWOULDBLOCK)
    {
      /* send failed. :-( */
      /* stay in same state, try again */
      x->state=REQUESTSEND;
      return 1;
    }
    fdset_close(&x->sock);
    /* try connection again. maybe to different host if >1 */
    debug_printf(("Send failed.(%d)\n",errno));
    http_error( no, error_soft, "Error sending request" );
    return 0;
  }
/*
  debug_printf(("send okay\n (sent : %s), size %d, ret %d, sock %d\n",request,size,ret,x->sock)); */
  x->state=RESPONSE;

  return 0;

}


static char *http_gethostname( int no, char *buf, int bufsize )
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ( (char *) httpfetch[no] ) + sizeof(httpfetch_t);

  bufsize=bufsize; /* avoid warning */

  if ( !x->proxy )
    http_setproxy(no);

  if ( x->proxy >= 2 && x->proxy <= 4 )
    return config_proxyname[ x->proxy - 2 ];

  if ( memcmp(url, "http:", 5 ) != 0 )
  {
    /* http is the only scheme we can handle without a proxy */
    http_error( no, error_hard, "Unprocessable scheme" );
    return NULL;
  }
  http_spliturl( url, buf);
  return buf;
}

/* http_init()
 *
 * called once a url has been added to a session, it then starts does something
 * (ie. resolve or connect, generally)
 * the 'retry' parameter should be set if this is a retry, and causes retries field to be preserved
 */
int http_init(int no, int retry)
{
  char h[HOST_MAXLEN];
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ( (char *) httpfetch[no] ) + sizeof(httpfetch_t);
  char *host;
  char *ptr;
  int reuse;

//  debug_printf(( "http_init : depth = %d\n", x->linkdepth ));
  status_seturl( no, url );

//   debug_printf(("Bufflen at init = %d\n",x->bufflen));
  host = http_gethostname( no, h, sizeof h );

  if ( !host )
    return 0;

//  debug_printf(("Proxy = %d, Comparing '%s' to '%s'\n",x->proxy,host,x->host));
  reuse = x->sock>=0 && my_strcaseequ(host,x->host);
  debug_printf(("Host = '%s', current = %s, sock = %d, reusing connection = %d\n", host,x->host,x->sock,reuse));

  /* Now, the only thing we could possible want out the old session
     is the socket, where and proxy fields....
     (and the retries field if we're retrying) */
  {
    int sock        = x->sock;
    URL_LISTS where = x->where;
    int proxy       = x->proxy;
    int retries     = retry ? x->retries-1 : 3;
    int linkdepthtogo = x->linkdepthtogo;
    int act_linkdepth = x->act_linkdepth;
    int fetchportion = x->fetchportion;
    int imagefetchportion = x->imagefetchportion;
    int noinlineimages = x->noinlineimages;
    int inlineframes = x->inlineframes;
    int ruleset      = x->ruleset;
    memset(httpfetch[no], 0, sizeof(*httpfetch[no]));
    x->sock      = sock;
    x->where     = where;
    x->proxy     = proxy;
    x->retries   = retries; /* Number of times to retry when something goes wrong */
    x->linkdepthtogo = linkdepthtogo;
    x->act_linkdepth = act_linkdepth;
    x->fetchportion = fetchportion;
    x->noinlineimages = noinlineimages;
    x->inlineframes = inlineframes;
    x->ruleset      = ruleset;
    x->imagefetchportion = imagefetchportion;
    strcpy(x->host,host); /* copy in new/old hostname */
  }

  if (reuse)
  {
    debug_printf(("%d : *** Re-using previous connection!***\n",no));
    /* we're already connected to the right place */
    return http_requestsend(no);
  }

  fdset_close(&httpfetch[no]->sock); /* shut any old connection! */

  ptr=strchr(host,':');
  if (ptr) *ptr=0;

//  debug_printf(("Looking up '%s'\n",x->host));

  x->dnsquery = dns_gethostbyname(host);
  if (ptr) *ptr=':'; /* in case there was a port number specified */

  // debug_printf(("sess->dnsquery = %p\n", x->dnsquery));
  if (x->dnsquery == NULL)
  {
    http_error( no, error_hard, "DNS start failed (no memory?)" );
    return 0;
  }

  httpfetch_setpoll(no); /* we require polling, whilst waiting for the dns to finish */

  x->state=RESOLVE;
  return 0;
}

static int http_resolvewait(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  struct hostent *host = NULL;
  int n;
  dns_status_t status;

/*   debug_printf(("resolve_wait, dnsquery = %p\n",x->dnsquery)); */

  if (!x->dnsquery) return -1; /* error. */

  status = dns_check(x->dnsquery);
  if ( status != dns_complete_success && status != dns_complete_failure )
    return 1; /* still waiting */
//  debug_printf(("status = %d (fail=%d,succ=%d)\n",status,dns_query_complete_failure,dns_query_complete_success));


  /* we know we've succeeded or failed now */
  if (status == dns_complete_success)
    host = dns_getanswer(x->dnsquery);

  dns_dispose(x->dnsquery);
  x->dnsquery = NULL;
  httpfetch_removepoll(no);

  if ( status == dns_complete_failure || !host || !host->h_addr_list )
  {
    char msg[256];
    char h[HOST_MAXLEN];
    char *host = http_gethostname( no, h, sizeof h );
    debug_printf(("Soft failure : dns lookup failed"));
    strcpy( msg, "Could not resolve hostname " );
    if ( host )
      strncat( msg, host, sizeof(msg)-sizeof("Could not resolve hostname "));
    http_error( no, error_soft, msg );
    return 0;
  }

  /* success. start connection */
  debug_printf(("%d : *** connecting to %s ***\n",no,x->host));
  n=socket_connect(host, socket_getport(HTTP_PORT,x->host));
  if (n<0) return n-2;
  x->sock = n;
  fdset_addwrite(x->sock);

//  debug_printf(("resolving done, connecting\n"));
  x->state=CONNECT;

  return 0; /* done waiting */
}



static int http_connectwait(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int size = sizeof(int);
  int error;

  /* read socket error. no error -> connected */
  if (getsockopt(x->sock, SOL_SOCKET, SO_ERROR, (char *) &error, &size)==-1)
  {
    debug_printf(("getsockopt returned error, %d\n",errno));
    http_error( no, error_soft, "Error calling getsockopt() waiting for connect" );
    return 0;
  }

  // debug_printf(("getsockopt - socket error = %d\n",error));
  if (error != 0)
  {
    char msg[ 100 ];
    debug_printf(("connect returned error, %d\n",error));
    fdset_close(&x->sock);
    sprintf( msg, "Could not connect (socket error %d)", error );
    http_error( no, error_soft, msg );
    return 0;
  }
  /* okay, no error -> we're connected. */
  fdset_movetoread(x->sock);

  /* send request */
//  debug_printf(("Sending request.....\n"));
  x->state=REQUESTSEND;

  return http_requestsend(no);
}


/* This is used to recalculate fetchportion when we either get redirected to a new
 * url, or when we have a base href.
 */

int http_calcfetchportion( const char *url, conf_followlinks links )
{
  const char *tmp;
  if ( links == links_all )
    return 0;

  if ( links == links_host )
  {
    int cnt = 0;
    tmp = url;
    while ( *tmp && *tmp !='/' )
    {
      /* find 3rd / or end of string, whichever comes first */
      if ( *tmp == '/' && ++cnt >= 3 )
        break;
      tmp++;
    }
    return tmp - url;
  }

  /* must be links_dir - fetch in same directory */
  tmp = strrchr( url, '/' ); /* go back to last '/' */
  if ( !tmp )
    return 0; /* no '/'!?!?, default to all urls. */

  return tmp - url;
}

int http_recalcfetchportion( const char *newurl, const char *oldurl, int oldp )
{
  if ( oldp == 0 )
    return 0; /* fetching all urls anyway */

  if ( oldp == http_calcfetchportion( oldurl, links_host ) )
    return http_calcfetchportion( newurl, links_host );

  return http_calcfetchportion( newurl, links_dir );
}


static int http_getdata(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int ret;
  int spare = sizeof(x->buffstart) - x->bufflen;

  if (spare <= 0) return -1; /* buffer full. must empty it */

  // debug_printf(("buffstart = %p, bufflen = %d",x->buffstart, x->bufflen));
//  debug_printf(("%d : http_getdata, spare = %d\n",no,spare));

  ret = recv( x->sock, x->buffstart+x->bufflen, spare, 0 );

  if ( ret < 0 )
  {
    if (errno==EWOULDBLOCK)
      return 0; /* no new data */
    else
      return -2; /* read error */
  }

  if (ret==0) return -3; /* connection closed */

  x->bufflen     += ret; /* real new data! */
  http_bytesrxed += ret;

  return ret; /* new data */
}

static void http_takedata(int no, int amount)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  /* decrease amount of data in buffer */
  x->bufflen-= amount;
  /* move up sockptr to point to next bit of data */
  if (x->bufflen>0) memmove(x->buffstart, x->buffstart+amount, x->bufflen);
}

static int file_ensuredir(char *name)
{
  _kernel_oserror *e;
  char *ptr=name;

  while ( (ptr=strchr(ptr,'.')) != NULL )
  {
    *ptr=0;
    e=_swix(OS_File, _INR(0,1) | _IN(4), 8, name, 0 /* ? - no. of entries */);
    *ptr='.';
    if (e) return -1; /* failed */
    ptr++;
  }
  return 1;
}


static int http_openoutput(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
  char name[FILEPATH_MAXLEN];
  int ret;
  FILE *f;

  if (x->html) return -2; /* already an open file. ooooops... */

  ret = misc_urltofilename( url, name, sizeof name );
  if (ret<0) return ret;

  ret = file_ensuredir(name);
  if (ret<0) { debug_printf(("Open failed for : %s\n",name));return ret;}

  /* check if it exists or summat... */
  f = fopen(name, "w");
  if (!f) {debug_printf(("Open failed for : %s\n",name));return -1;} /* file open failed, permanent, I think */

  x->html = f;
  return 0;
}


static void http_fetchrelativeurl(int no, char *reference, eval_inline inline)
{
  char result[URL_MAXLEN], *ans;
  char *url = ((char *) httpfetch[no]) + sizeof(httpfetch_t);
  char *ptr;
  int ret;

  if ( httpfetch[no]->baseurl )
  {
    /* resolve against base href */
    debug_printf(("Resolving '%s' against '%s' : ", reference, url+strlen(url)+1 ));
    ans = misc_resolverelativeurl( url + strlen( url ) + 1, reference, result, sizeof(result) - 1 );
    debug_printf(("'%s'\n", ans ? ans : "(null)" ));
  }
  else
    ans = misc_resolverelativeurl( url, reference, result, sizeof(result) - 1 );

  if ( !ans ) return;

  /* remove trailing and leading spaces */
  while ( isspace(*ans) ) ans++;
  ptr = ans + strlen(ans) - 1;
  while ( isspace(*ptr) ) *ptr--=0;

  ret = url_handled( ans );
  if ( ret == 0 )
  {
    eval_url( no, ans, inline );
    /* NB. flex blocks may have shifted now */
  }
  else if ( ret > 0 )
  {
    /* ftp and no proxy set */
    http_adderrorlog( ans, "No ftp proxy set" );
  }
  else
  {
    /* unrecognised url (ignore mailto's & javascript) */
    if ( !my_strncaseequ( ans, "mailto:", sizeof("mailto:")-1 ) &&
         !my_strncaseequ( ans, "javascript:", sizeof("javascript:")-1 )    )
      http_adderrorlog( ans, "No handler for scheme" );
  }

}




static int http_processtag(int no, int len)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *lookfrom;
  eval_inline inline = inline_not;
  int basehref = 0;

  lookfrom = misc_scantag( x->tagbuff, len, &basehref, &inline, NULL );
  if (!lookfrom) return 0;

  debug_printf(("Tag '%s' is a possible fetch\n",x->tagbuff));

  if ( basehref )
  {
    http_addbasehref( no, lookfrom );
    return 1;
  }

  http_fetchrelativeurl(no, lookfrom, inline);

  /* NB. flex blocks may have shifted now */
  return 1;
}


static int http_checkfortags(int no, int amount)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *ptr = x->buffstart;
  int copy=0;
  char *tag = x->tagbuff + strlen(x->tagbuff);
  int maxtags = 75;

//  debug_printf(("*** Check for tags, amount = %d\n",amount));

  if (!amount) return 0; /* nothing to process */

  if (tag!=x->tagbuff) copy=1;

  do
  {
    if (*ptr=='<')
    {
      /* Start of tag. Abort any present one, start copying */
      tag = x->tagbuff;
      *tag=0;
      copy=1;
    }
    else if (*ptr=='>')
    {
      /* End of tag. Are we copying? Yes -> process*/
      if (copy)
      {
//        int p;
        int offset = ptr - x->buffstart; /* record offset... block may move */
        *tag=0;
        if ( http_processtag( no, tag - x->tagbuff ) ) maxtags--;
        /* we may have moved the flex blocks... have to reset all pointers */
        x = httpfetch[no]; /* NB. Don't pass x to flex & co */
        ptr = x->buffstart + offset;
        tag = x->tagbuff;
//        if ( p )
//          debug_printf(("Processed '%s', maxtags = %d\n", x->tagbuff,maxtags));
        *tag=0;
        copy=0;
        if ( maxtags <= 0 )
        {
          amount--;
          httpfetch_setpoll( no );
          break;
        }
      }
    }
    else if (copy)
    {
      if (tag >= x->tagbuff+sizeof(x->tagbuff))
      {
        /* Buffer full. balls. */
        tag = x->tagbuff;
        *tag=0;
        copy=0;
      }
      else
        *tag++=*ptr;
    }
  } while (ptr++, --amount > 0);
  *tag=0; /* Add terminator so we know where we got to next time! */

  return amount; /* number of unprocessed bytes */
}



/* Get document body.
   Two possible finish points.
   * Close of connection
   * Known length, and we've got that much data
 */

/* This routine has two purposes in life :
   * Sink everything arriving from the server (eg. body of errors, etc)
   * Write contents out to file. If it's html, scan for tags
     * Scanning :
         Don't take data that looks like it starts a tag.
         Unless.... it's the end of a doc and no more data will arrive.
 */
static int http_bodywait(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int ret, written;
  int forus;
  int cnt = 5;
  int incomplete = 0;

  do
  {
    ret = http_getdata(no);

    if (x->encoding==CONTENTLENGTH || x->encoding==CHUNKED)
      forus = MIN( x->length - x->rx_bytes, x->bufflen );
    else
      forus = x->bufflen;


    if (x->html)
    {
      if (x->parse)
      {
        int ret = http_checkfortags(no, forus); /* correct if we only parsed a small amount of the text */
        x = httpfetch[no]; /* flex blocks may have moved... NB. Don't pass x to flex & co */
        if ( ret ) /* if unprocessed data */
        {
          forus -= ret;
          cnt = 0; /* don't process any more */
          incomplete = 1;
        }
        else
          if ( x->poll ) httpfetch_removepoll( no );
      }

      written=fwrite(x->buffstart,1,forus,x->html);
      if (written!=forus)
      {
        http_error( no, error_hard, "Unable to write to html file" );
        return 0;
      }
    }
    x->rx_bytes += forus;
    http_takedata( no,forus );
    if ( incomplete ) return 1;
//    debug_printf(("ret = %d, forus = %d, rx_bytes = %d, length = %d, x->bufflen = %d\n",ret,forus,x->rx_bytes,x->length,x->bufflen));

    if (x->rx_bytes==x->length && (x->encoding==CONTENTLENGTH || x->encoding==CHUNKED))
    {
      char tmp[10];
      if (x->encoding==CHUNKED)
      { /* End of chunk */
        x->state=HEADER; /* You can have header *AFTER* chunks! D'oh! */
        return 0;
      }
      if ( ! ( x->retries <= 0 && ( recv( x->sock, tmp, sizeof tmp, MSG_PEEK ) > 0 || x->bufflen > 0 ) ) )
      {
        http_fetchfinished( no, NULL ); /* Document complete - reached correct length */
        return 0;
      }
      /* uh-oh. data remaining, we're on our last retry.
       * chances are the server lied. let's wait for connection close instead */
      debug_printf(("%d : Server returned data after end of page, trying to forgive it\n",no));
      x->encoding = UNKNOWN;
      continue; /* go back and process any data we didn't think was for us */
    }

    if (ret<=-2) /* hmm. connection broken, probably. */
    {
      if (x->encoding == UNKNOWN && ret == -3) /* 'Graceful' close */
      {
        debug_printf(("%d : End of doc!(no encoding and other end closed connection)\n",no));
        http_fetchfinished( no, NULL ); /* other end closed connection, expected doc. length unknown */
        return 0;
      }
      debug_printf(("%d : getdata returned error : %d\n",no,ret));
      if ( ret == -3 )
      {
        debug_printf(( "%d : Other end closed connection. We expected %d, we got %d.(encode=%d)\n",
                no,x->length,x->rx_bytes,x->encoding));
      }
      debug_printf(("%d : errno = %d, expected = %d, got = %d, encode = %d\n",
                      no,errno,x->length,x->rx_bytes,x->encoding));
      http_error( no, error_forgiveable, "Connection broken before whole document received" ); /* retrying. dunno? */
      return 0;
    }
  } while ( forus > 0 && cnt-- > 0 );

  return 1;
}

static int http_log( int no, char *ptr )
{
  int fail = 0;

  if ( !http_logf )
    http_logf = fopen( LOG_DIR".FetchLog", "a" );

  if ( !http_logf )
    return -1;

  {
    char *url = ((char *) httpfetch[no]) + sizeof(httpfetch_t); /* NB. no flex calls */
    int resp = httpfetch[no]->respcode;
    char datestring[80];
    time_t date = time(NULL);
    strftime( datestring, sizeof datestring, "%a %b %d %H:%M:%S", localtime(&date) );
    if (ptr)
      fail = fprintf( http_logf, "%s %s %3d %s\n", datestring, url, resp, ptr ) < 0;
    else
      fail = fprintf( http_logf, "%s %s %3d\n",    datestring, url, resp ) < 0;
    if ( fail < 0 )
    {
      fclose( http_logf );
      http_logf = fopen( LOG_DIR".FetchLog", "a" );
      if ( !http_logf ) return -1;
      if (ptr)
        fail = fprintf( http_logf, "%s %s %3d %s\n", datestring, url, resp, ptr ) < 0;
      else
        fail = fprintf( http_logf, "%s %s %3d\n",    datestring, url, resp ) < 0;

      if ( fail < 0 ) return -1;
    }
    return 0;
  }
}


static const char http_months[13][4] =
{
  "Jan", "Feb", "Mar", "Apr", "May", "Jun",
  "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", ""
};

static time_t http_parsedate( char *date )
{
  struct tm out;
  time_t output;
  /** Parsing date:
     Find time as two colons with one or two digits inbetween
     Year is four ajcent digits
     Day is first non-time number
     Year is first non-time non-day number
     Month is string search for one of 3 digit months
  **/
  char *time = date;
  char *month = date, *day, *year;
//  debug_printf(( "Last mod : '%s'\n", date ));
  memset( &out, 0, sizeof out );

  /* search for xx:xx:xx */
  do
  {
    time = strchr( time, ':' );
    if ( !time ) { debug_printf((" No colon!\n" )); return 0; }
    time -= 2;
    if ( time>=date && isdigit( time[0] ) && isdigit( time[1] )
         && isdigit( time[3] ) && isdigit( time[4] ) && time[5]==':'
         && isdigit( time[6] ) && isdigit( time[7] ) )
      break;
    time += 3; /* skip over ':' */
  }
  while (1);

#define dig( x ) ( x - '0' )
  /* time now points at xx:xx:xx, where isdigit(x) */
  out.tm_sec  = dig( time[6] ) * 10 + dig( time[7] );
  out.tm_min  = dig( time[3] ) * 10 + dig( time[4] );
  out.tm_hour = dig( time[0] ) * 10 + dig( time[1] );
  out.tm_mon = 0;

  while ( *http_months[ out.tm_mon ] )
  {
    month = strstr( date, http_months[ out.tm_mon ] );
    if ( month ) break;
    out.tm_mon++;
  }

  if ( !*http_months[ out.tm_mon ] ) return 0;

  /* day of month is first day either to left of month, or closest on right */
  day = month;
  while ( day >= date && !isdigit( *day ) ) day--;

  out.tm_mday = 0;
  if ( isdigit( *day ) )
  {
    /* found a digit to the left, find the start of the number & convert to decimal */
    while ( day > date && isdigit( *(day-1) ) ) day--;
    /* date points at first digit, convert to decimal */
    while ( isdigit( *day ) )
      out.tm_mday = out.tm_mday * 10 + dig( *day++ );
  }

  /* whether we find a day to the left or not, we expect the first thing to the right to be a year
   * (skipping the time)
   */
  year = month;
  while ( *year && !isdigit( *year ) ) year++;

  if ( !*year ) return 0; /* couldn't find year */
  /* assume this is the year */
  out.tm_year = 0;
  while ( isdigit( *year ) )
    out.tm_year = out.tm_year * 10 + dig( *year++ );
  /* years 0-69 are in 20xx.
     year 70-99 are 19xx.
     Take others at face value (struct tm specifices years since 1900) */
  if ( out.tm_year <= 69 )
    out.tm_year += 2000;
  else if ( out.tm_year <= 99 )
    out.tm_year += 1900;

  out.tm_year -= 1900;

// NB. This code does not skip time & year
//  if ( out.tm_mday == 0 )
//  {
//    /* search for a digit right, abort if we don't find one */
//    day = month;
//    while ( *day && !isdigit( *day ) ) day++;
//    if ( !*day ) return 0;
//  }
/*  debug_printf(("Input to mktime:\n"));
  debug_printf(("sec = %d\n", out.tm_sec));
  debug_printf(("min = %d\n", out.tm_min));
  debug_printf(("hour= %d\n", out.tm_hour));
  debug_printf(("mday= %d\n", out.tm_mday));
  debug_printf(("mon = %d\n", out.tm_mon));
  debug_printf(("year= %d\n", out.tm_year));*/
  output = mktime( &out );
  if ( output == (time_t) -1 ) { debug_printf(("mktime called failed :-(\n")); return 0; }
//  debug_printf(( "Parsed date is '%s'\n", ctime( &output ) ));
  return output;
}
#undef dig


static int http_parseheader(int no, int linelen)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  char *ptr;

  if ( x->encoding!=CHUNKED && my_strstarts(x->buffstart, linelen, "Content-length:") )
  {
    ptr = x->buffstart + sizeof("Content-Length:") - 1;
    while (!isdigit(*ptr) && *ptr) ptr++;
    if (*ptr)
    {
      x->length   = atoi(ptr);
      x->encoding = CONTENTLENGTH;
      // debug_printf(("Content-Length = %d\n",x->length));
    }
  }
  /* Warning - The contents of the buffer are *NOT* null terminated... */
  if ( my_strstarts(x->buffstart,linelen, "Transfer-Encoding:") )
  {
    ptr = x->buffstart + sizeof("Transfer-Encoding:") - 1;
    linelen-=sizeof("Transfer-Encoding:") - 1;
    while (*ptr==' ' || *ptr=='\t') { ptr++; linelen--; }
    // debug_printf(("ptr = %s",ptr));
    // debug_printf(("line len = %d\n",linelen));
    if (my_strstarts(ptr,linelen, "chunked"))
    {
      x->encoding = CHUNKED;
      // debug_printf(("Data is chunked.\n"));
    }
  }
  if ( my_strstarts(x->buffstart, linelen, "Content-Type:") )
  {
    char *p;
    ptr = x->buffstart + sizeof("Content-Type:") - 1;
    linelen-=sizeof("Content-Type:") - 1;
    /* We know point at the Content-Type. Get the 'ref' for this type */
    *(ptr+linelen)=0;
    if (p = strchr(ptr, ';'), p)
      /* might be Content-Type: text/html; charset=blah */
      *p = 0;
    ptr = my_strstrip(ptr); /* removing leading / trailing white space */
    x->filetype = mimetype_getfiletype( ptr );
    if (!strcmp(ptr,"text/html") && x->respcode==200)
      x->parse = 1; /* Look for links */
//    debug_printf(("Looked up '%s', Adding mimetype to fetch %d = %p\n",ptr,no,result));
  }
//   if ( my_strstarts(x->buffstart, linelen, "Date:") )
//   {
//     ptr = x->buffstart + sizeof("Date:") - 1;
//     linelen -= sizeof("Date:") - 1;
//     *(ptr+linelen)=0;
//     ptr = my_strstrip(ptr); /* removing leading / trailing white space */
//     x->date = http_parsedate( ptr );
//   }
  if ( my_strstarts(x->buffstart, linelen, "Last-Modified:") )
  {
    ptr = x->buffstart + sizeof("Last-Modified:") - 1;
    linelen -= sizeof("Last-Modified:") - 1;
    *(ptr+linelen)=0;
    ptr = my_strstrip(ptr); /* removing leading / trailing white space */
    debug_printf(("%d : Last-Modified = '%s'\n", no, ptr ));
    x->date = http_parsedate( ptr );
  }
  if ( (x->respcode==301 || x->respcode==302) && my_strstarts(x->buffstart, linelen, "Location:") )
  {
    char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
    char result[URL_MAXLEN], *absurl;
    int fetchp, imagefetchp;
    ptr = x->buffstart + sizeof("Location:") - 1;
    linelen -= sizeof("Location:") - 1;
    /* Record the new url */
    *(ptr+linelen)=0;
    ptr = my_strstrip(ptr);
    absurl = misc_resolverelativeurl(url, ptr, result, sizeof(result) - 1);
    if (!absurl)
      return 0;
    /* fetch the url we're redirected to */
    fetchp = http_recalcfetchportion( absurl, url, x->fetchportion );
    imagefetchp = http_recalcfetchportion( absurl, url, x->imagefetchportion );
    url_addto( PENDING, absurl, x->linkdepthtogo, fetchp , x->noinlineimages, x->inlineframes, x->ruleset,
               inline_not /*hmm*/, x->act_linkdepth, imagefetchp );
    http_log( no, ptr ); /* add redirection entry to log */
    {
      int handle_redirection_like_base_href;
    }
  }

  return 0;
}

/* returns:
   0 - document over
   1 - still working
 */
static int http_headerover(int no)
{
          /* Responses that MUST NOT include a body
             to any HEAD command
             1xx (informational)
             204 (no content)
             304 (not modified)

            Special action : 3xx -  redirections:
               300 - multiple choices...???
               301 - Permanent move, record url as name given....
               302 - Temporary move, record url as orig. one, but download from tmp loc
              -> Look for new location field!
               304 - not modified. for conditional GET!
               305 - must use resource given in location: header as proxy to get url
           */
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  /* blank line. body or chunk starts here. or, we're done if it's a HEAD or similar */

  switch (x->respcode)
  {
    case 200:
      /* OK. either get the page, or update the header info, or something */
      if ( x->rx_bytes==0 )
      {
        /* So we don't open the file again whilst in the middle of chunks */
        if ( http_openoutput(no) < 0 )
        {
          char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
          http_adderrorlog( url, "File open failed" );
          debug_printf(("fileopen failed!\n"));
        }
      }
      break;
/*    case 404:
      debug_printf(("%d : Page not found... oh dear!\n",no));
      break;
    case 500:
      debug_printf(("%d : Server error... oh dear!\n",no));
      break; */
    case 304:
      debug_printf(("%d : Not modified!\n",no));
      break;
  }
  if (x->respcode!=200) debug_printf(("%d : respcode = %d\n",no,x->respcode));

  if (x->where==FOUND || (x->respcode>=100 && x->respcode<=199) || x->respcode==204 || x->respcode==304)
  {
    /* No body. erm. */
    if (x->respcode==200)
    {
      int UPDATE_LIST;
      E_REPORT(("Need to update entry in found list / move to pending\n"));
      exit(0);
    }
    else if (x->respcode==304)
    {
      debug_printf(("%d : page unchanged\n",no));
//      http_log( no, NULL );
//      http_fetchfinished( no );
      return 0;
    }
    else
    {
      int no_content_etc;
      E_REPORT("no_content_etc");
      exit(0);
    }
  }
  else /* Setup for recv'ing body */
    x->state = (x->encoding==CHUNKED) ? CHUNK : BODY; /* unknown encoding -> body */



  return 1;
}

/* return values. -ve -> error
   0 = state changed. +ve -> processing? */
#define LINESTATE(x) ((x)==RESPONSE || (x)==HEADER || (x)==CHUNK || (x)==CHUNKFOOT)
static int http_linewait(int no)
{
  httpfetch_t *x = httpfetch[no]; /* NB. Don't pass x to flex & co */
  int ret;
  char *ptr;
  int cnt = 0;

  ret = http_getdata(no);
  /* process return code after any data... */

  {
    int FIXME_CONTINUATION_LINE;
    /* if a line starts ' ' or '\t', it is a continuation line :/ */
  }
  while ( ((ptr=memchr(x->buffstart, '\n', x->bufflen)) != NULL || ret==-1) && LINESTATE(x->state) )
  {
    int linelen = ptr ? (ptr) - x->buffstart : x->bufflen;
    int amount  = ptr ? linelen+1            : x->bufflen; /* include \n, unless full buffer! */

    /* while there's a newline, or the buffer is full */
    if (!ptr)
    {
      debug_printf(("Buffer is full!!!!\n"));
      ptr=x->buffstart+x->bufflen-1; /* buffer must be full... */
    }

    if ( *(x->buffstart+linelen-1)=='\r' ) linelen--;

    switch (x->state)
    {
      case RESPONSE:
//        debug_printf(("Response(%d):\n",linelen));
/*        fwrite(x->buffstart, 1, linelen, stdout);
        debug_printf(("\n")); */
      {
        /* rip out the response code */
        char *ptr = memchr(x->buffstart,' ',linelen);
        if ( ptr && isdigit( *(ptr+1) ) )
        {
          x->respcode = atoi(ptr+1);
          if ( x->respcode == 404 || x->respcode == 500 )
          {
            /* log these now, so we can get anything interesting the server said */
            FILE *f = http_errlogf;
            if (f)
            {
              char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
              time_t date = time(NULL);
              char datestring[80];
              strftime(datestring, sizeof datestring, "%a %b %d %H:%M:%S", localtime(&date));
              x->buffstart[ linelen ] = 0;
              fprintf(f,"%s %s http error : %s\n", datestring, url, ptr+1);
            }
            else
            {
              debug_printf(("%d : unable to write to log file\n",no));
            }
          }
        }
        else
        {
          /* If there's no space, or it's not followed by a digit, let's assume http
             0.9. Make sure we don't take any data, and jump to BODY state.
             How do we flag that the transfer should be terminated by connection
             close? x->encoding==UNKNOWN? */
          amount=0;
          x->state=BODY;
          if ( http_openoutput(no) < 0 )
          {
            char *url = ((char *) httpfetch[no])+sizeof(httpfetch_t); /* NB. no flex calls */
            http_adderrorlog( url, "File open failed" );
            debug_printf(("%d : fileopen failed!\n",no)); /* output should just get sunk by code */
          }
          x->encoding=UNKNOWN;
          break;
        }
        x->state=HEADER;
      } break;

      case HEADER:
//        debug_printf(("Header = '"));
//        fwrite(x->buffstart, 1, linelen, stdout);
//        debug_printf(("'\n"));
        if (linelen==0) /* End of header. Start body, or act... depends on response code */
        {
          if ( http_headerover(no) <= 0 )
          {
            /* doc has finished. (eg. in case of 304 not modified) */
            debug_printf(("%d : end of headers -> end of doc\n", no ));
            http_takedata(no, amount);
            http_fetchfinished( no, NULL ); /* end of header with 304 response etc. */
            return 0;
          }
          break;
        }
        http_parseheader(no, linelen);
        break;

      case CHUNKFOOT: /* We don't know much about these... no use to us */
//        debug_printf(("Chunkfoot:"));
        if (linelen==0) /* End of doc */
        {
          http_takedata(no, amount);
//          debug_printf(("End of chunk footers! Doc complete!!!!!\n"));
          http_fetchfinished( no, NULL ); /* end of chunk footers after last chunk */
          return 0;
        }
        break;

      case CHUNK:
//        debug_printf(("Chunk:"));
        // *(x->buffstart + linelen + 1)=0;
        // debug_printf(("Chunk '%s'\n",x->buffstart));
        x->length = my_hexatoi(x->buffstart);
//        debug_printf(("Size of this chunk is %d\n",x->length));
        if (x->length==0)
        {
//          debug_printf(("Doc complete, getting chunk footers\n"));
//          debug_printf(("Bufflen = %d, amount = %d\n",x->bufflen, amount));
          http_takedata(no, amount);
          x->state=CHUNKFOOT;
          return 0;
        }
        x->rx_bytes=0;
        x->state=BODY;
	break;

      default:
        debug_printf(("Argh. Unknown state in line_wait()"));
    }
    // debug_printf(("taking data : bufflen = %d, amount = %d\n",x->bufflen, amount));
    http_takedata(no, amount);
    if ( cnt++ > 5 ) return 0;
  } /* while */
  if ( ret<0 && !(ret==-1 && errno==EWOULDBLOCK) && LINESTATE(x->state) )
  {
    /* An error occurred. We can't get connections closed whilst we're expecting a header,
    so something must have gone wrong. */
    char error[128];
    sprintf( error, "Socket error (%d,%d) receiving header", ret, errno );
    debug_printf(( "%d : ret = %d, errno = %d\n", no, ret, errno ));
    http_error( no, error_soft, error );
    return 0;
  }
  return 0;
}



int http_pollsess(int no, int error)
{
  int ret = -1, laststate;

  error = error;

  do
  {
    laststate = httpfetch[no]->state;
    //  debug_printf(("polling : %d, state=%d - ",no,x->state));
    switch ( httpfetch[no]->state )
    {
      case RESOLVE:
        //        debug_printf(("Resolve:\n"));
        ret = http_resolvewait(no);
        break;
      case CONNECT:
        //      debug_printf(("Connect:\n"));
        ret = http_connectwait(no);
        break;
      case REQUESTSEND:
        //    debug_printf(("Request send:\n"));
        ret = http_requestsend(no);
        break;
      case RESPONSE:
      case HEADER:
      case CHUNK:
      case CHUNKFOOT:
        //  debug_printf(("Response wait:\n"));
        ret = http_linewait(no);
        break;
      case BODY:
        //debug_printf(("Body wait:\n"));
        ret = http_bodywait(no);
        break;
      default:
        http_error( no, error_hard, "State machine entered unknown state" );
        break;
    }
    if ( ret == -1 )
    {
      debug_printf(("Error - perm fail\n"));
      http_error( no, error_hard, "Unknown hard error" );
    }
    else if ( ret < 0 )
    {
      debug_printf(("Error : %d\n",ret));
      http_error( no, error_soft, "Unknown soft error" );
    }
  } while ( laststate != httpfetch[no]->state && httpfetch[no]->state != CONNECT && !httpfetch[no]->idle );
            /* handle any data that might already be there for the next state */

  return -1;
}


int http_openlog( void )
{
  http_logf = fopen( LOG_DIR".FetchLog", "a" );
  http_errlogf = fopen( LOG_DIR".FetchError", "a" );
  if ( !http_logf || !http_errlogf )
  {
    if ( http_logf )
    {
      fclose( http_logf );
      http_logf = 0;
    }
    if ( http_errlogf )
    {
      fclose( http_errlogf );
      http_errlogf = 0;
    }
    return -1;
  }

  return 0;
}


void http_closelog( void )
{
  if ( http_logf )
  {
    fclose( http_logf );
    http_logf = NULL;
  }
  if ( http_errlogf )
  {
    fclose( http_errlogf );
    http_errlogf = NULL;
  }
  url_clear(); /* empty all the url lists */
}

