/*
** Copyright (C) 1995, Enterprise Integration Technologies Corp.        
** All Rights Reserved.
** Kevin Hughes, kevinh@eit.com 
** 3/11/94
**
** Released under the GPL by EIT
**
** Heavily hacked for Harvest
*/

#include "swish.h"
#include "merge.h"
#include "error.h"
#include "file.h"
#include "search.h"
#include "string.h"
#include "index.h"
#include "hash.h"
#include "mem.h"

/* The main merge functions - it accepts three file names.
** This is a bit hairy. It basically acts as a zipper,
** zipping up both index files into one.
*/

void readmerge(file1, file2, outfile)
     char *file1;
     char *file2;
     char *outfile;
{
	int i, j, indexfilenum1, indexfilenum2, result, totalfiles,
		skipwords, skipfiles;
	long limit1, limit2, fileinfo1, fileinfo2, offsetstart;
	char line[MAXSTRLEN];
	struct indexentry *ip1, *ip2, *ip3;
	struct indexentry *buffer1, *buffer2;
	FILE *fp1, *fp2, *fp3;
	struct metaMergeEntry *metaFile1, *metaFile2;

	/* remapVar is used into addindexfilelist and need to be
	** initialized each time two indexes are merged.
	*/
	remapVar = 0;
	metaFile1 = metaFile2 = NULL;

	if ((fp1 = fopen(file1, "r")) == NULL) {
		sprintf(errorstr, "Couldn't read the index file \"%s\".",
		file1);
		progerr(errorstr);
	}
	if (!isokindexheader(fp1)) {
		sprintf(errorstr, "\"%s\" has an unknown format.",
		file1);
		progerr(errorstr);
	}
	if ((fp2 = fopen(file2, "r")) == NULL) {
		sprintf(errorstr, "Couldn't read the index file \"%s\".",
		file2);
		progerr(errorstr);
	}
	if (!isokindexheader(fp2)) {
		sprintf(errorstr, "\"%s\" has an unknown format.",
		file2);
		progerr(errorstr);
	}

	ip1 = ip2 = ip3 = NULL;
	buffer1 = buffer2 = NULL;
	if (verbose)
		printf("Counting files... ");
	indexfilenum1 = getindexfilenum(fp1);
	indexfilenum2 = getindexfilenum(fp2);
	totalfiles = indexfilenum1 + indexfilenum2;
	if (verbose) {
		printf("%d files.\n", indexfilenum1 + indexfilenum2);
		printf("Reading stopwords...");
	}
	readoffsets(fp1);
	readstopwords(fp1);
	limit1 = offsets[STOPWORDPOS];
	fileinfo1 = offsets[FILELISTPOS];
	metaFile1 = readMergeMeta(metaFile1,fp1);

	readoffsets(fp2);
	readstopwords(fp2);
	limit2 = offsets[STOPWORDPOS];
	fileinfo2 = offsets[FILELISTPOS];
	metaFile2 = readMergeMeta(metaFile2,fp2);

        /* Create the merged list and modify the
	   individual ones with the new meta index
        */	   
	metaEntryList = createMetaMerge(metaFile1, metaFile2);

	if (verbose)
		printf("\nReading file info...");

	totalfiles=indexfilenum1+indexfilenum2;
/*
	fseek(fp1, fileinfo1, 0);
	for (i = 1; i <= indexfilenum1; i++) {
		fgets(line, MAXSTRLEN, fp1);
		addindexfilelist(i, line, &totalfiles);
	}
	fseek(fp2, fileinfo2, 0);
	for (i = 1; i <= indexfilenum2; i++) {
		fgets(line, MAXSTRLEN, fp2);
		addindexfilelist(i + indexfilenum1, line, &totalfiles);
	}
*/

	if ((fp3 = fopen(outfile, "w")) == NULL) {
		sprintf(errorstr,
		"Couldn't write the merged index file \"%s\".",
		outfile);
		progerr(errorstr);
	}

	if (verbose)
		printf("\nMerging words... ");

	printheader(fp3, outfile, 0, totalfiles);

        offsetstart = ftell(fp3);
        for (i = 0; i < MAXCHARS; i++)
                fprintf(fp3, "%016li", offsets[i]);
        fputc('\n', fp3);

	readoffsets(fp1);
	readoffsets(fp2);

	for (i = 0; i < MAXCHARS; i++)
		offsets[i] = 0;

	skipwords = 0;
	while (1) {
		if (buffer1 == 	NULL) {
			ip1 = readindexline(fp1, limit1,metaFile1);
			if (ip1 == NULL) {
			  if (ip2 == NULL) {
			    break;
			  }
			}
			buffer1 = ip1;
		}
		if (buffer2 == NULL) {
			ip2 = readindexline(fp2, limit2,metaFile2);
			if (ip2 == NULL){
			  if (ip1 == NULL) {
			    break;
			  }
			}
			else 
			  addfilenums(ip2, indexfilenum1);
			buffer2 = ip2;
		}
		if (ip1 == NULL)
		  result = 1;
		else if (ip2 == NULL)
		  result = -1;
		else
		  result = strcmp(ip1->word, ip2->word);
		if (!result) {
			ip3 = mergeindexentries(ip1, ip2);
			printindexentry(ip3, fp3);
			freeindexentry(ip1);
			freeindexentry(ip2);
			freeindexentry(ip3);
			ip1 = ip2 = ip3 = NULL;
			buffer1 = buffer2 = NULL;
			skipwords++;
		}
		else if (result < 0) {
			printindexentry(ip1, fp3);
			freeindexentry(ip1);
			buffer1 = NULL;
			ip1 = NULL;
		}
		else {
			printindexentry(ip2, fp3);
			freeindexentry(ip2);
			buffer2 = NULL;
			ip2 = NULL;
		}
	}

	if (verbose) {
		if (skipwords)
			printf("%d redundant word%s.", skipwords,
			(skipwords == 1) ? "" : "s");
		else
			printf("no redundant words.");
	}

        printstopwords(fp3);

	if (verbose)
		printf("\nMerging file info... ");

	offsets[FILELISTPOS] = ftell(fp3);

	fseek(fp1, fileinfo1, 0);
	for (i= 1; i <= indexfilenum1; i++) {
		addtofilehashlist(i-1, ftell(fp3));
		fgets(line, MAXSTRLEN, fp1);
		fputs(line, fp3);
	}
	fseek(fp2, fileinfo2, 0);
	for (i= 1; i <= indexfilenum2; i++) {
		addtofilehashlist(i+indexfilenum1-1, ftell(fp3));
		fgets(line, MAXSTRLEN, fp2);
		fputs(line, fp3);
	}
/*
	for (i = j = 1; i <= indexfilenum1 + indexfilenum2; i++)
		if (getmap(i) == j) {
			addtofilehashlist(j++ - 1, ftell(fp3));
			fprintf(fp3, "%s", lookupindexfilenum(i));
		}
*/
	skipfiles = (indexfilenum1 + indexfilenum2) - totalfiles;
	if (verbose) {
		if (skipfiles)
			printf("%d redundant file%s.", skipfiles,
			(skipfiles == 1) ? "" : "s");
		else
			printf("no redundant files.");
	}
	printfileoffsets(fp3);
	printMetaNames(fp3);

	fseek(fp3, offsetstart, 0);
        for (i = 0; i < MAXCHARS; i++)
                fprintf(fp3, "%016li", offsets[i]);
	fclose(fp3);

	fclose(fp1);
	fclose(fp2);

	if (verbose)
		printf("\nDone.\n");
}

/* Gets the number of files in an index file.
*/

int getindexfilenum(fp)
     FILE *fp;
{
	int i;
	char line[MAXSTRLEN];

	readoffsets(fp);
	fseek(fp, offsets[FILELISTPOS], 0);

	i = 0;
	while(ftell(fp) != offsets[FILEOFFSETPOS]) {
		fgets(line, MAXSTRLEN, fp);
		i++;
	}

	return i;
}

/* This adds an offset to the file numbers in a particular
** result list. For instance, file 1 has file numbers going from
** 1 to 10, but so does file 2, so I have to add 10 to all the
** file numbers in file 2 before merging.
*/

void addfilenums(ip, num)
     struct indexentry *ip;
     int num;
{
	struct resultMerge *rp;

	rp = ip->result;
	while (rp != NULL) {
		rp->filenum = encodefilenum(decodefilenum(rp->filenum) + num);
		rp = rp->next;
	}
}

/* This reads the next line in the index file and puts the results
** in a result structure.
*/

struct indexentry *readindexline(FILE *fp, long limit, 
				 struct metaMergeEntry * metaFile)
{
        int i, c, x, countnum, rank=0, filenum=0, attribute;
        char fileword[MAXWORDLEN];
        struct resultMerge *rp;
	struct indexentry *ip;
	struct metaMergeEntry* tmp;

        rp = NULL;

	if (limit == ftell(fp))
		return NULL;
        for (i = 0; (c = fgetc(fp)) != 0; ) {
                if (c == ':') {
                        fileword[i] = '\0';
                        break;
                }
                else
                        fileword[i++] = c;
        }

        countnum = 1;

        ungetc(c, fp);
        while ((c = fgetc(fp)) != 0) {
                x = 0;
                do {
                        c = fgetc(fp);
                        if (c == 0)
                                break;
                        x *= 128;
                        x += c & 127;
                } while (c & 128);
		if (c == 0)
			break;
                if (x) {
                        if (countnum == 1) {
                                filenum = x;
                                countnum++;
                        }
                        else if (countnum == 2) {
                                rank = x;
                                countnum++;
                        }
                        else if (countnum == 3) {
			        attribute = x;
				/*Need to modify metaName with new list*/
				for(tmp=metaFile;tmp;tmp=tmp->next) {
				  if (tmp->oldIndex == attribute) {
				    attribute = tmp->newIndex;
				    break;
				  }
				}
                                rp = addtoresultlistMerge(rp, filenum,
				rank, attribute);
				countnum = 1;
                        }
                }
        }

	ip = (struct indexentry *) emalloc(sizeof(struct indexentry));
	ip->word = (char *) mystrdup(fileword);
	ip->result = rp;

        return ip;
}


/* This simply concatenates two information lists that correspond
** to a word found in both index files.
*/

struct indexentry *mergeindexentries(struct indexentry *ip1, 
				     struct indexentry *ip2)
{
	struct resultMerge *newrp, *rp1, *rp2;
	struct indexentry *ep;

	rp1 = ip1->result;
	rp2 = ip2->result;
	newrp = NULL;

	while (rp1 != NULL) {
		newrp = addtoresultlistMerge(newrp, rp1->filenum, rp1->rank, 
					     rp1->attribute);
		rp1 = rp1->next;
	}
	while (rp2 != NULL) {
		newrp = addtoresultlistMerge(newrp, rp2->filenum, rp2->rank, 
					     rp2->attribute);
		rp2 = rp2->next;
	}

	ep = (struct indexentry *) emalloc(sizeof(struct indexentry));
	ep->word = (char *) mystrdup(ip1->word);
	ep->result = newrp;

	return ep;
}

/* This prints a new word entry into the merged index file,
** removing redundant file information as it goes along.
*/

void printindexentry(ip, fp)
     struct indexentry *ip;
     FILE *fp;
{
	int i, num, attribute;
	struct resultMerge *rp;

	for (i = 0; indexchars[i] != '\0'; i++)
		if ((ip->word)[0] == indexchars[i] && !offsets[i])
			offsets[i] = ftell(fp);
	fprintf(fp, "%s:", ip->word);
	rp = ip->result;
	while (rp != NULL) {
		num = rp->filenum;
		attribute = rp->attribute;
		compress(num, fp);
		compress(rp->rank, fp);
		compress(rp->attribute,fp);
		rp = rp->next;
	}
	fputc(0, fp);
}

#ifdef NDEF
/* This associates a number with a new number.
** This function is used to remap file numbers from index
** files to a new merged index file.
*/

void remap(oldnum, newnum)
     int oldnum;
     int newnum;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct mapentry *mp;

	mp = (struct mapentry *) emalloc(sizeof(struct mapentry));
	mp->oldnum = oldnum;
	mp->newnum = newnum;

	sprintf(tmpstr, "%d", oldnum);
	hashval = bighash(tmpstr);
	mp->next = mapentrylist[hashval];
	mapentrylist[hashval] = mp;
}


/* This retrieves the number associated with another.
*/


int getmap(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct mapentry *mp;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp = mapentrylist[hashval];

	while (mp != NULL) {
		if (mp->oldnum == num)
			return mp->newnum;
		mp = mp->next;
	}
	return num;
}
#endif


/* This marks a number as having been printed.
*/

void marknum(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct markentry *mp;

	mp = (struct markentry *) emalloc(sizeof(struct markentry));
	mp->num = num;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp->next = markentrylist[hashval];
	markentrylist[hashval] = mp;
}

#ifdef NDEF
/* Same thing but for merge only */
void marknumMerge(num, attribute)
     int num;
     int attribute;
{
        unsigned hashval;
        char tmpstr[MAXSTRLEN];
        struct markentryMerge *mp;
 
        mp = (struct markentryMerge *) emalloc(sizeof(struct markentryMerge));
        mp->num = num;
	mp->attribute = attribute;
 
        sprintf(tmpstr, "%d", num);
        hashval = bighash(tmpstr);
        mp->next = markentrylistMerge[hashval];
        markentrylistMerge[hashval] = mp;
}      
#endif

/* Has a number been printed?
*/

int ismarked(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct markentry *mp;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp = markentrylist[hashval];

	while (mp != NULL) {
		if (mp->num == num)
			return 1;
		mp = mp->next;
	}
	return 0;
}

#ifdef NDEF
int ismarkedMerge(int num,int attribute)
{
        unsigned hashval;
        char tmpstr[MAXSTRLEN];
        struct markentryMerge *mp;
 
        sprintf(tmpstr, "%d", num);
        hashval = bighash(tmpstr);
        mp = markentrylistMerge[hashval];
 
        while (mp != NULL) {
                if ( (mp->num == num) && (mp->attribute == attribute) )
                        return 1;
                mp = mp->next;
	      }
        return 0;
      }
#endif

/* Initialize the marking list.
*/

void initmarkentrylist()
{
	int i;
	struct markentry *mp;

	for (i = 0; i < BIGHASHSIZE; i++) {
		mp = markentrylist[i];
		if (mp != NULL)
			free(mp);
		markentrylist[i] = NULL;
	}
}

#ifdef NDEF
void initmarkentrylistMerge()
{
        int i;
        struct markentryMerge *mp;
 
        for (i = 0; i < BIGHASHSIZE; i++) {
                mp = markentrylistMerge[i];
                if (mp != NULL)
                        free(mp);
                markentrylistMerge[i] = NULL;
	      }
}      


/* Initialize the main file list.
*/

void initindexfilehashlist()
{
  int i;
  struct indexfileinfo *ip;
  
  for (i = 0; i < BIGHASHSIZE; i++) {
    ip = indexfilehashlist[i];
    if (ip != NULL)
      free(ip);
    indexfilehashlist[i] = NULL;
  }
}

/* Initialize the mapentrylist 
*/

void initmapentrylist()
{
  int i;
  struct mapentry *ip;
  
  for (i = 0; i < BIGHASHSIZE; i++) {
    ip = mapentrylist[i];
    if (ip != NULL)
      free(ip);
    mapentrylist[i] = NULL;
  }
}

#endif

/* Frees up used index entries, my best attempt at memory management...
** I still have bytes leaking elsewhere...
*/

void freeindexentry(ip)
     struct indexentry *ip;
{
	struct resultMerge *rp, *oldp;

	free(ip->word);
	rp = ip->result;
	while (rp != NULL) {
		oldp = rp;
		rp = rp->next;
		free(oldp);
	}
	free(ip);
}

/* Translates a file number into something that can be compressed.
*/

int encodefilenum(num)
     int num;
{
	int i, j;

	for (i = j = 0; i != num; i++) {
		j++;
		if (!(j % 128))
			j++;
	}
	return j;
}

/* Translates a compressed file number into a correct file number.
*/

int decodefilenum(num)
     int num;
{
        int i, extra;

        for (i = 1, extra = 0; i < num; i++)
                if (!(i % 128)) {
                        extra++;
                        i++;
                }
        num -= extra;

	return num;
}

/* Similar to addtoresultlist, but also adding the meta name
*/

struct resultMerge *addtoresultlistMerge(struct resultMerge *rp, 
					 int filenum, int rank,int attribute)
{
        struct resultMerge *newnode;
        static struct resultMerge *head;
 
        newnode = (struct resultMerge *) emalloc(sizeof(struct resultMerge));
        newnode->filenum = filenum;
        newnode->rank = rank;
	newnode->attribute = attribute;
        newnode->next = NULL;
 
        if (rp == NULL)
                rp = newnode;
        else
                head->next = newnode;
 
        head = newnode;
 
        return rp;
}

/* Reads the meta names from the index. Needs to be different from
** readMetaNames because needs to zero out the counter.
*/
struct metaMergeEntry* readMergeMeta(metaFile,fp)
     struct metaMergeEntry* metaFile;
     FILE* fp;
{     
  int i, c, counter;
  char word[MAXWORDLEN];
  
  counter = 0;
  fseek(fp, offsets[METANAMEPOS], 0);
  for (i = 0; (c = fgetc(fp)) != '\n' && c != EOF; ){
    if (!isspace(c))
      word[i++] = c;
    else {
      word[i] = '\0';
      metaFile = addMetaMerge(metaFile, word,&counter);
      i = 0;
    }
  }
  return metaFile;
}

/* Adds an entry to the list of meta names for one index, 
** setting the new index to 0 - it will then be set by 
** createMetaMerge.
*/

struct metaMergeEntry* addMetaMerge(metaFile, metaWord,counter)
     struct metaMergeEntry* metaFile;
     char* metaWord;
     int* counter;
{
  int i;
  struct metaMergeEntry* newEntry;
  struct metaMergeEntry* tmpEntry;
  
  if (*counter == 0)
    *counter = 2;
  else if ((*counter) == 1 ||  (!((*counter) % 128)) )
   (*counter)++;
  for( i=0; metaWord[i]; i++)
    metaWord[i] =  tolower(metaWord[i]);
  newEntry = (struct metaMergeEntry*) emalloc(sizeof(struct metaMergeEntry));
  newEntry->metaName = (char*)mystrdup(metaWord);
  newEntry->oldIndex = (*counter)++;
  newEntry->newIndex = 0;
  newEntry->next = NULL;
  if (metaFile)
    {
      for(tmpEntry=metaFile;tmpEntry->next!=NULL;tmpEntry=tmpEntry->next)
        ;
      tmpEntry->next = newEntry;
    }
  else
    metaFile = newEntry;
  
  return metaFile;
}

/* Creates a list of all the meta names in the indexes
*/
struct metaEntry* createMetaMerge(metaFile1, metaFile2)
     struct metaMergeEntry* metaFile1;
     struct metaMergeEntry* metaFile2;
{
  struct metaMergeEntry* tmpEntry;
  int counter;

  metaEntryList = NULL;
  counter = 0;
  for (tmpEntry=metaFile1;tmpEntry;tmpEntry=tmpEntry->next)
    metaEntryList = addMetaMergeList(metaEntryList,tmpEntry,&counter);
  
  for (tmpEntry=metaFile2;tmpEntry;tmpEntry=tmpEntry->next)
    metaEntryList = addMetaMergeList(metaEntryList,tmpEntry,&counter);

  return metaEntryList;
}

/* Adds an entry to the merged meta names list and changes the
 ** new index in the idividual file entry
 */

struct metaEntry* addMetaMergeList(metaEntryList,metaFileEntry,count)
     struct metaEntry* metaEntryList;
     struct metaMergeEntry* metaFileEntry;
     int* count;
{
  int i, wordExists, newIndex;
  struct metaEntry* newEntry;
  struct metaEntry* tmpEntry;
  struct metaEntry* last;
  char *metaWord, *compWord;
  
  wordExists = 0;
  if ((*count) == 0)
    *count = 2;
  else if ((*count) == 1 ||  (!((*count) % 128)) )
    (*count)++;
  metaWord = metaFileEntry->metaName;
  
  for( i=0; metaWord[i]; i++)
    metaWord[i] =  tolower(metaWord[i]);
  if (metaEntryList)
    {
      for(tmpEntry=metaEntryList;tmpEntry;tmpEntry=tmpEntry->next)
	{
	  if (tmpEntry->next == NULL)
	    last = tmpEntry;
	  compWord = tmpEntry->metaName;
	  if (!strcmp(compWord,metaWord) ) 
	    {
	      wordExists = 1;
	      newIndex = tmpEntry->index;
	      break;
	    }
	}
      if (wordExists)
	metaFileEntry->newIndex = newIndex;
      else {
	newEntry = (struct metaEntry*) emalloc(sizeof(struct metaEntry));
	newEntry->metaName = (char*)mystrdup(metaWord);
	newEntry->index = *count;
	newEntry->next = NULL;
	metaFileEntry->newIndex = (*count)++;
	last->next = newEntry;
      }
    }
  else {
    newEntry = (struct metaEntry*) emalloc(sizeof(struct metaEntry));
    newEntry->metaName = (char*)mystrdup(metaWord);
    newEntry->index = *count;
    newEntry->next = NULL;
    metaFileEntry->newIndex = (*count)++;
    metaEntryList = newEntry;
  }
  return metaEntryList;
}