/************************************************************************************************* * CGI script to merge results of searchers * Copyright (C) 2003-2005 Mikio Hirabayashi * This file is part of Estraier, a personal full-text search system. * Estraier is free software; you can redistribute it and/or modify it under the terms of the GNU * General Public License as published by the Free Software Foundation; either version 2 of the * License, or any later version. * Estraier is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU General Public License for more details. * You should have received a copy of the GNU General Public License along with Estraier; * if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, * MA 02111-1307 USA. *************************************************************************************************/ #include "estcommon.h" #define AGENTNAME "Estraier" /* name of the client program */ #define CONFFILE "estmerge.conf" /* name of the merger file */ #define ENCODING "UTF-8" /* encoding of the page */ #define SAVENAME "estresult.html" /* name of the file to be saved */ #define RDATAMAX 262144 /* max size of data to read */ #define MAXWORDS 128 /* max number of search words */ #define EXTRAMAX 4 /* extra of max paremeter for children */ #define SOCKBUFSIZ 8192 /* size of buffer for socket */ typedef struct { /* type of structure for each word */ char *label; /* the label */ char *word; /* the word */ int hit; /* number of hits */ } METAWORD; typedef struct { /* type of structure for each document */ int score; /* matching score */ char *uri; /* expression of the URI */ char *label; /* expression of the label */ char *title; /* expression of the title */ char *author; /* expression of the author */ char *date; /* expression of the date */ char *type; /* expression of the type */ char *enc; /* expression of the encoding */ char *size; /* expression of the size */ char *relsc; /* expression of the scores */ char *summary; /* expression of the summary */ } METADOC; /* global variables for configurations */ const char *scriptname = NULL; /* name of the script */ CBLIST *tlabels = NULL; /* list of target labels */ CBLIST *turls = NULL; /* list of target URLs */ const char *pxhost = NULL; /* host name of the proxy */ int pxport = 0; /* port number of the proxy */ const char *tmplfile = NULL; /* path of the template file */ const char *topfile = NULL; /* path of the top page file */ int hidetl = FALSE; /* whether to hide target labels */ int defmax = 0; /* number of shown documents by default */ const char *logfile = NULL; /* path of the log file for input search conditions */ const char *ldocs = NULL; /* label label shared among all documents */ const char *lperpage = NULL; /* label of the select box for showing number */ const char *ldrep = NULL; /* label of the select box for directory rep level */ const char *lsortsc = NULL; /* label of the option of sorting by score */ const char *lsortdt = NULL; /* label of the option of sorting by date */ const char *lreverse = NULL; /* label of the option of reverse sorting */ const char *lexasis = NULL; /* label of the option of as-is expressions */ const char *lexwild = NULL; /* label of the option of expressions with wild cards */ const char *lexregex = NULL; /* label of the option of regular expressions */ const char *lsubmit = NULL; /* label of the submit button to search */ const char *lresult = NULL; /* label of the result */ const char *lhits = NULL; /* label of suffix of the hit number */ const char *lrelated = NULL; /* label of a link to search related documents */ const char *lprev = NULL; /* label of the link to go to the previous page */ const char *lnext = NULL; /* label of the link to go to the next page */ const char *luri = NULL; /* label for the attribute of URI */ const char *ltitle = NULL; /* label for the attribute of title */ const char *lauthor = NULL; /* label for the attribute of author */ const char *ldate = NULL; /* label for the attribute of date */ const char *ltype = NULL; /* label for the attribute of type */ const char *lenc = NULL; /* label for the attribute of encoding */ const char *lsize = NULL; /* label for the attribute of size */ const char *enohit = NULL; /* error message when no document hits */ const char *enotarget = NULL; /* error message when no target was selected */ const char *enoword = NULL; /* error message when no effective word */ /* global variables for parameters */ const char *phrase = NULL; /* search phrase */ int max = 0; /* number of shown documents */ int drep = 0; /* level of directory rep */ const char *sort = NULL; /* sorting order */ const char *expr = NULL; /* expression class */ int page = 0; /* numerical order of the page */ const char *mrglbl = NULL; /* label assigned by mergers */ int showsc = FALSE; /* whether to show score expression */ const char *relsc = NULL; /* score map of seed document */ int *checks = NULL; /* array of checkboxes for targets */ const char *chkstr = NULL; /* parameter string for targets */ /* other global variables */ int tabidx = 0; /* counter of tab indexes */ /* function prototypes */ int main(int argc, char **argv); const char *skiplabel(const char *str); CBMAP *getparams(void); void showerror(const char *msg); void xmlprintf(const char *format, ...); void showform(void); void showresult(void); CBLIST *geturldata(const CBLIST *urls); CBLIST *getsockdata(const int *fds, int fdn); const char *mygethostaddr(const char *host); int mygettcpconn(const char *addr, unsigned short port); int sockwrite(int fd, const void *buf, int size); void extractinfo(const char *body, METAWORD *words, int *wnp, int wmax, METADOC *docs, int *dnp, int dmax); int mdoccompare(const void *a, const void *b); int mdocrcompare(const void *a, const void *b); void showdoc(const METADOC *docp, int num); void showversion(void); void outputlog(void); /* main routine */ int main(int argc, char **argv){ CBLIST *lines, *elems; CBMAP *params; const char *tmp, *ep; char *esc, *tmpltext, *toptext, key[ESTNUMBUFSIZ], *wp, *pname, *pbuf; int i, j, psiz; /* set configurations */ cbstdiobin(); scriptname = argv[0]; if((tmp = getenv("SCRIPT_NAME")) != NULL) scriptname = tmp; if((tmp = strrchr(scriptname, '/')) != NULL) scriptname = tmp + 1; if(!(lines = cbreadlines(CONFFILE))) showerror("the configuration file is missing."); cbglobalgc(lines, (void (*)(void *))cblistclose); tlabels = cblistopen(); cbglobalgc(tlabels, (void (*)(void *))cblistclose); turls = cblistopen(); cbglobalgc(turls, (void (*)(void *))cblistclose); for(i = 0; i < cblistnum(lines); i++){ tmp = cblistval(lines, i, NULL); if(cbstrfwimatch(tmp, "target:")){ tmp = skiplabel(tmp); if(!(ep = strchr(tmp, '@'))) showerror("the target expression is invalid."); cblistpush(tlabels, tmp, ep - tmp); cblistpush(turls, ep + 1, -1); } else if(cbstrfwimatch(tmp, "proxy:")){ tmp = skiplabel(tmp); if(tmp[0] != '\0'){ if(!(ep = strchr(tmp, ':'))) showerror("the proxy expression is invalid."); esc = cbmemdup(tmp, ep - tmp); cbglobalgc(esc, free); pxhost = esc; if((pxport = atoi(ep + 1)) < 1) showerror("the proxy expression is invalid."); } } else if(cbstrfwimatch(tmp, "tmplfile:")){ tmplfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "topfile:")){ topfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "hidetl:")){ if(!cbstricmp(skiplabel(tmp), "true")) hidetl = TRUE; } else if(cbstrfwimatch(tmp, "defmax:")){ defmax = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "logfile:")){ logfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldocs:")){ ldocs = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lperpage:")){ lperpage = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldrep:")){ ldrep = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsortsc:")){ lsortsc = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsortdt:")){ lsortdt = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lreverse:")){ lreverse = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexasis:")){ lexasis = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexwild:")){ lexwild = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexregex:")){ lexregex = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsubmit:")){ lsubmit = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lresult:")){ lresult = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lhits:")){ lhits = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lrelated:")){ lrelated = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lprev:")){ lprev = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lnext:")){ lnext = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "luri:")){ luri = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ltitle:")){ ltitle = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lauthor:")){ lauthor = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldate:")){ ldate = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ltype:")){ ltype = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lenc:")){ lenc = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsize:")){ lsize = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enohit:")){ enohit = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enotarget:")){ enotarget = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enoword:")){ enoword = skiplabel(tmp); } } if(cblistnum(tlabels) < 1) showerror("no target is defined."); if(!tmplfile) showerror("tmplfile is undefined."); if(!topfile) showerror("topfile is undefined."); if(defmax < 1) showerror("defmax is undefined."); if(!logfile) logfile = ""; if(!ldocs) ldocs = ""; if(!lperpage) showerror("lperpage is undefined."); if(!ldrep) showerror("ldrep is undefined."); if(!lsortsc) showerror("lsortsc is undefined."); if(!lsortdt) showerror("lsortdt is undefined."); if(!lreverse) showerror("lreverse is undefined."); if(!lexasis) showerror("lexasis is undefined."); if(!lexwild) showerror("lexwild is undefined."); if(!lexregex) showerror("lexregex is undefined."); if(!lsubmit) showerror("lsubmit is undefined."); if(!lresult) showerror("lresult is undefined."); if(!lhits) showerror("lhits is undefined."); if(!lrelated) showerror("lrelated is undefined."); if(!lprev) showerror("lprev is undefined."); if(!lnext) showerror("lnext is undefined."); if(!luri) showerror("luri is undefined."); if(!ltitle) showerror("ltitle is undefined."); if(!lauthor) showerror("lauthor is undefined."); if(!ldate) showerror("ldate is undefined."); if(!ltype) showerror("ltype is undefined."); if(!lenc) showerror("lenc is undefined."); if(!lsize) showerror("lsize is undefined."); if(!enohit) showerror("enohit is undefined."); if(!enotarget) showerror("enotarget is undefined."); if(!enoword) showerror("enoword is undefined."); /* read other files */ if(!(tmpltext = cbreadfile(tmplfile, NULL))) showerror("the template file is missing."); cbglobalgc(tmpltext, free); if(!(toptext = cbreadfile(topfile, NULL))) showerror("the top page file is missing."); cbglobalgc(toptext, free); /* read parameters */ params = getparams(); cbglobalgc(params, (void (*)(void *))cbmapclose); phrase = cbmapget(params, "phrase", -1, NULL); if((tmp = cbmapget(params, "max", -1, NULL)) != NULL) max = atoi(tmp); if((tmp = cbmapget(params, "drep", -1, NULL)) != NULL) drep = atoi(tmp); sort = cbmapget(params, "sort", -1, NULL); expr = cbmapget(params, "expr", -1, NULL); if((tmp = cbmapget(params, "page", -1, NULL)) != NULL) page = atoi(tmp); mrglbl = cbmapget(params, "mrglbl", -1, NULL); if((tmp = cbmapget(params, "showsc", -1, NULL)) != NULL && !strcmp(tmp, "true")) showsc = TRUE; relsc = cbmapget(params, "relsc", -1, NULL); if(!phrase) phrase = ""; if(max < 1) max = defmax; if(drep < 0) drep = 0; if(!sort) sort = "score"; if(!expr) expr = "asis"; if(page < 1) page = 1; if(!mrglbl) mrglbl = ""; if(!relsc) relsc = ""; if(relsc[0] != '\0'){ sort = "score"; expr = "asis"; } if((tmp = cbmapget(params, "enc", -1, NULL)) != NULL){ if((esc = cbiconv(phrase, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); phrase = esc; } if((esc = cbiconv(mrglbl, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); mrglbl = esc; } if((esc = cbiconv(relsc, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); relsc = esc; } } if(cbstrfwimatch(phrase, "[relsc] ")){ esc = cburldecode(strchr(phrase, ' ') + 1, NULL); cbglobalgc(esc, free); relsc = esc; phrase = ""; sort = "score"; expr = "asis"; } checks = cbmalloc(cblistnum(tlabels) * sizeof(int) + 1); cbglobalgc(checks, free); esc = cbmalloc(cblistnum(tlabels) * ESTNUMBUFSIZ + 1); cbglobalgc(esc, free); wp = esc; if((tmp = cbmapget(params, "t0", -1, NULL)) != NULL){ wp += sprintf(wp, "&t0=on"); for(i = 0; i < cblistnum(tlabels); i++){ checks[i] = TRUE; } } else { for(i = 0; i < cblistnum(tlabels); i++){ checks[i] = FALSE; sprintf(key, "t%d", i + 1); if((tmp = cbmapget(params, key, -1, NULL)) != NULL && !strcmp(tmp, "on")){ wp += sprintf(wp, "&t%d=on", i + 1); checks[i] = TRUE; } } } *wp = '\0'; chkstr = esc; if(cbmaprnum(params) < 1){ for(i = 0; i < cblistnum(tlabels); i++){ checks[i] = TRUE; } } /* send HTTP headers */ printf("Cache-Control: no-cache, must-revalidate, no-transform\r\n"); printf("Pragma: no-cache\r\n"); printf("Content-Disposition: inline; filename=%s\r\n", SAVENAME); printf("Content-Type: text/html; charset=%s\r\n", ENCODING); printf("\r\n"); /* parse the template */ elems = cbxmlbreak(tmpltext, FALSE); cbglobalgc(elems, (void (*)(void *))cblistclose); for(i = 0; i < cblistnum(elems); i++){ tmp = cblistval(elems, i, NULL); if(!strcmp(tmp, "")){ printf("
\n"); showform(); printf("
"); fflush(stdout); } else if(!strcmp(tmp, "")){ printf("
\n"); if(phrase[0] != '\0' || relsc[0] != '\0'){ showresult(); } else { printf("%s", toptext); } printf("
"); fflush(stdout); } else if(!strcmp(tmp, "")){ printf("
\n"); showversion(); printf("
"); fflush(stdout); } else if(cbstrfwmatch(tmp, "")){ pname = cbmemdup(tmp + 12, strlen(tmp) - 15); if((pbuf = cbreadfile(pname, &psiz)) != NULL){ for(j = 0; j < psiz; j++){ putchar(pbuf[j]); } free(pbuf); } free(pname); fflush(stdout); } else if(cbstrfwmatch(tmp, "")){ estputenv("ESTPHRASE", phrase); pname = cbmemdup(tmp + 12, strlen(tmp) - 15); if((pbuf = estreadexec(pname, &psiz)) != NULL){ for(j = 0; j < psiz; j++){ putchar(pbuf[j]); } free(pbuf); } free(pname); fflush(stdout); } else { while(*tmp != '\0'){ if(*tmp == '{'){ if(cbstrfwmatch(tmp, "{ESTPHRASE}")){ xmlprintf("%@", phrase); tmp += 11; } else if(cbstrfwmatch(tmp, "{ESTMAX}")){ xmlprintf("%d", max); tmp += 8; } else { putchar(*tmp); tmp++; } } else { putchar(*tmp); tmp++; } } } } /* output log */ outputlog(); /* exit normally */ return 0; } /* skip the label of a line */ const char *skiplabel(const char *str){ if(!(str = strchr(str, ':'))) return str; str++; while(*str != '\0' && (*str == ' ' || *str == '\t')){ str++; } return str; } /* get a map of the CGI parameters */ CBMAP *getparams(void){ CBMAP *params; CBLIST *pairs; char *rbuf, *buf, *key, *val, *dkey, *dval; const char *tmp; int i, len, c; params = cbmapopen(); rbuf = NULL; buf = NULL; if((tmp = getenv("CONTENT_LENGTH")) != NULL && (len = atoi(tmp)) > 0 && len <= RDATAMAX){ rbuf = cbmalloc(len + 1); for(i = 0; i < len && (c = getchar()) != EOF; i++){ rbuf[i] = c; } rbuf[i] = '\0'; if(i == len) buf = rbuf; } else { buf = getenv("QUERY_STRING"); } if(buf != NULL){ buf = cbmemdup(buf, -1); pairs = cbsplit(buf, -1, "&"); for(i = 0; i < cblistnum(pairs); i++){ key = cbmemdup(cblistval(pairs, i, NULL), -1); if((val = strchr(key, '=')) != NULL){ *(val++) = '\0'; dkey = cburldecode(key, NULL); dval = cburldecode(val, NULL); cbmapput(params, dkey, -1, dval, -1, FALSE); free(dval); free(dkey); } free(key); } cblistclose(pairs); free(buf); } free(rbuf); return params; } /* show the error page and exit */ void showerror(const char *msg){ printf("Status: 500 Internal Server Error\r\n"); printf("Content-Type: text/plain; charset=%s\r\n", ENCODING); printf("\r\n"); printf("Error: %s\n", msg); exit(1); } /* XML-oriented printf */ void xmlprintf(const char *format, ...){ va_list ap; char *tmp; unsigned char c; va_start(ap, format); while(*format != '\0'){ if(*format == '%'){ format++; switch(*format){ case 's': tmp = va_arg(ap, char *); if(!tmp) tmp = "(null)"; printf("%s", tmp); break; case 'd': printf("%d", va_arg(ap, int)); break; case '@': tmp = va_arg(ap, char *); if(!tmp) tmp = "(null)"; while(*tmp){ switch(*tmp){ case '&': printf("&"); break; case '<': printf("<"); break; case '>': printf(">"); break; case '"': printf("""); break; default: putchar(*tmp); break; } tmp++; } break; case '?': tmp = va_arg(ap, char *); if(!tmp) tmp = "(null)"; while(*tmp){ c = *(unsigned char *)tmp; if((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c != '\0' && strchr("_-.", c))){ putchar(c); } else { printf("%%%02X", c); } tmp++; } break; case '%': putchar('%'); break; } } else { putchar(*format); } format++; } va_end(ap); } /* show index information */ void showform(void){ int i; xmlprintf("
\n", scriptname); xmlprintf("
\n"); xmlprintf("\n"); if(relsc[0] != '\0'){ xmlprintf("\n", relsc, ++tabidx); } else { xmlprintf("\n", phrase, ++tabidx); } xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n", lsubmit, ++tabidx); xmlprintf("\n"); xmlprintf("
\n"); xmlprintf("
\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("\n"); xmlprintf("
\n"); xmlprintf("
\n"); if(hidetl){ xmlprintf("\n"); } else { for(i = 0; i < cblistnum(tlabels); i++){ xmlprintf("\n"); xmlprintf("\n", i + 1, checks[i] ? " checked=\"checked\"" : "", i + 1, ++tabidx, i + 2); xmlprintf("%@", cblistval(turls, i, NULL), phrase, max, drep, sort, expr, relsc, cblistval(tlabels, i, NULL)); xmlprintf("\n"); } } xmlprintf("
\n"); xmlprintf("
\n"); } /* show search result */ void showresult(void){ METAWORD *words; METADOC *docs; CBLIST *urls, *data; CBMAP *uwords; const char *mime, *tmp, *label; char *escph, *esclb, *escsc, *url, *body, numbuf[ESTNUMBUFSIZ]; int i, j, cnum, wmax, wnum, dmax, dnum, all, sum, first; wmax = MAXWORDS * cblistnum(turls) * 2; words = cbmalloc(wmax * sizeof(METAWORD) + 1); wnum = 0; dmax = (max * page + EXTRAMAX) * cblistnum(turls) * 2; docs = cbmalloc(dmax * sizeof(METADOC) + 1); dnum = 0; urls = cblistopen(); cnum = 0; for(i = 0; i < cblistnum(turls); i++){ if(!checks[i]) continue; cnum++; escph = cburlencode(phrase, -1); esclb = cburlencode(cblistval(tlabels, i, NULL), -1); escsc = cburlencode(relsc, -1); url = cbsprintf("%s?phrase=%s&max=%d&drep=%d&sort=%s&expr=%s&tfidf=false" "&mrglbl=%s&t0=on&showsc=true&relsc=%s", cblistval(turls, i, NULL), escph, max * page + EXTRAMAX, drep, sort, expr, esclb, escsc); cblistpush(urls, url, -1); free(url); free(escsc); free(esclb); free(escph); } data = geturldata(urls); for(i = 0; i < cblistnum(data); i++){ mime = cblistval(data, i, NULL); if(!(tmp = strchr(mime, '\n'))) continue; mime = tmp + 1; body = cbmimebreak(mime, -1, NULL, NULL); extractinfo(body, words, &wnum, wmax, docs, &dnum, dmax); free(body); } xmlprintf("
\n"); xmlprintf("
%@:
\n", lresult); xmlprintf("
"); label = ldocs[0] != '\0' ? ldocs : mrglbl; all = 0; uwords = cbmapopen(); for(i = 0; i < wnum; i++){ if(words[i].word){ sum = 0; if((tmp = cbmapget(uwords, words[i].word, -1, NULL)) != NULL) sum = atoi(tmp); sum += words[i].hit; sprintf(numbuf, "%d", sum); cbmapput(uwords, words[i].word, -1, numbuf, -1, TRUE); } else { all += words[i].hit; } } if(cbmaprnum(uwords) > 0){ if(label[0] != '\0') xmlprintf("%@:", label); cbmapiterinit(uwords); for(i = 0; (tmp = cbmapiternext(uwords, NULL)) != NULL; i++){ if(i > 0) xmlprintf(" / "); xmlprintf("%@" " (%@)", i, tmp, i, cbmapget(uwords, tmp, -1, NULL)); } xmlprintf(" = "); xmlprintf("%d %@", all, lhits); } else if(cnum < 1){ xmlprintf("%@", enotarget); all = -1; } else if(relsc[0] != '\0'){ if(label[0] != '\0') xmlprintf("%@:", label); xmlprintf("%d %@", all, lhits); } else { xmlprintf("%@", enoword); all = -1; } cbmapclose(uwords); xmlprintf("
\n"); if(!hidetl){ for(i = 0; i < wnum; i++){ if(!(words[i].word)){ label = words[i].label; xmlprintf("
"); xmlprintf("%@ : ", label); first = TRUE; for(j = 0; j < wnum; j++){ if(!(words[j].word) || strcmp(words[j].label, label)) continue; if(!first) xmlprintf(" / "); first = FALSE; xmlprintf("%@" " (%d)", words[j].word, words[j].hit); } if(relsc[0] == '\0') xmlprintf(" = "); xmlprintf("%d %@", words[i].hit, lhits); xmlprintf("
\n"); } } } xmlprintf("
\n"); if(dnum < 1 && all != -1) xmlprintf("

%@

\n", enohit); if(cbstrfwmatch(sort, "r-")){ cbqsort(docs, dnum, sizeof(METADOC), mdocrcompare); } else { cbqsort(docs, dnum, sizeof(METADOC), mdoccompare); } for(i = (page - 1) * max; i < dnum && i < page * max; i++){ showdoc(docs + i, i + 1); } xmlprintf("
"); if(page > 1){ xmlprintf("" "%s", scriptname, phrase, max, drep, sort, expr, page - 1, chkstr, relsc, ++tabidx, lprev); } else { xmlprintf("%s", lprev); } xmlprintf(" "); if(page * max < dnum){ xmlprintf("" "%s", scriptname, phrase, max, drep, sort, expr, page + 1, chkstr, relsc, ++tabidx, lnext); } else { xmlprintf("%s", lnext); } xmlprintf("
"); for(i = 0; i < dnum; i++){ free(docs[i].uri); free(docs[i].label); free(docs[i].title); free(docs[i].author); free(docs[i].date); free(docs[i].type); free(docs[i].enc); free(docs[i].size); free(docs[i].summary); } for(i = 0; i < wnum; i++){ free(words[i].label); free(words[i].word); } cblistclose(data); cblistclose(urls); free(docs); free(words); } /* get data of URLs */ CBLIST *geturldata(const CBLIST *urls){ CBLIST *reqs, *hosts, *auths, *data; CBMAP *elems; CBDATUM *iobuf; const char *url, *scheme, *host, *port, *auth, *path, *query, *addr, *req; char *tmp; int i, *fds, fdn, pnum; fds = cbmalloc(cblistnum(urls) * sizeof(int) + 1); fdn = 0; reqs = cblistopen(); hosts = cblistopen(); auths = cblistopen(); for(i = 0; i < cblistnum(urls); i++){ url = cblistval(urls, i, NULL); elems = cburlbreak(url); scheme = cbmapget(elems, "scheme", -1, NULL); host = cbmapget(elems, "host", -1, NULL); port = cbmapget(elems, "port", -1, NULL); auth = cbmapget(elems, "authority", -1, NULL); path = cbmapget(elems, "path", -1, NULL); query = cbmapget(elems, "query", -1, NULL); if(!port) port = "80"; if(!query) query = ""; if(pxhost){ addr = mygethostaddr(pxhost); pnum = pxport; } else { addr = host ? mygethostaddr(host) : NULL; pnum = atoi(port); } if(scheme && !strcmp(scheme, "http") && host && path && path[0] == '/' && pnum > 0 && addr != NULL){ if((fds[fdn] = mygettcpconn(addr, pnum)) != -1){ fdn++; if(pxhost){ tmp = cbsprintf("http://%s:%s%s%s%s", host, port, path, query[0] != '\0' ? "?" : "", query); } else { tmp = cbsprintf("%s%s%s", path, query[0] != '\0' ? "?" : "", query); } cblistpush(reqs, tmp, -1); cblistpush(hosts, host, -1); cblistpush(auths, auth ? auth : "", -1); free(tmp); } else { xmlprintf("

%@(%d): connection refused.

\n", addr, pnum); } } else { xmlprintf("

%@: invalid URL or unknown host.

\n", url); } cbmapclose(elems); } for(i = 0; i < fdn; i++){ req = cblistval(reqs, i, NULL); host = cblistval(hosts, i, NULL); auth = cblistval(auths, i, NULL); iobuf = cbdatumopen("", 0); cbdatumcat(iobuf, "GET ", -1); cbdatumcat(iobuf, req, -1); cbdatumcat(iobuf, " HTTP/1.0\r\n", -1); cbdatumcat(iobuf, "Host: ", -1); cbdatumcat(iobuf, host, -1); cbdatumcat(iobuf, "\r\n", -1); cbdatumcat(iobuf, "Connection: close\r\n", -1); if(auth[0] != '\0'){ tmp = cbbaseencode(auth, -1); cbdatumcat(iobuf, "Authorization: Basic ", -1); cbdatumcat(iobuf, tmp, -1); cbdatumcat(iobuf, "\r\n", -1); free(tmp); } cbdatumcat(iobuf, "Cache-Control: no-cache, must-revalidate\r\n", -1); cbdatumcat(iobuf, "Pragma: no-cache\r\n", -1); cbdatumcat(iobuf, "User-Agent: " AGENTNAME "/" _EST_VERSION "\r\n", -1); cbdatumcat(iobuf, "\r\n", -1); sockwrite(fds[i], cbdatumptr(iobuf), cbdatumsize(iobuf)); cbdatumclose(iobuf); } data = getsockdata(fds, fdn); for(i = 0; i < fdn; i++){ shutdown(fds[i], SHUT_RDWR); close(fds[i]); } cblistclose(auths); cblistclose(hosts); cblistclose(reqs); free(fds); return data; } /* get data of reading sockets */ CBLIST *getsockdata(const int *fds, int fdn){ CBLIST *data; CBDATUM **each; char buf[SOCKBUFSIZ]; int i, *lfds, fdmax, rsiz; fd_set rfds; each = cbmalloc(sizeof(CBDATUM *) * fdn + 1); for(i = 0; i < fdn; i++){ each[i] = cbdatumopen("", 0); } lfds = cbmalloc(fdn * sizeof(int) + 1); for(i = 0; i < fdn; i++){ lfds[i] = fds[i]; } while(TRUE){ FD_ZERO(&rfds); fdmax = -1; for(i = 0; i < fdn; i++){ if(lfds[i] != -1){ FD_SET(lfds[i], &rfds); fdmax = lfds[i] > fdmax ? lfds[i] : fdmax; } } if(fdmax == -1) break; if(select(fdmax + 1, &rfds, NULL, NULL, NULL) == -1){ if(errno == EINTR) continue; break; } for(i = 0; i < fdn; i++){ if(lfds[i] != -1 && FD_ISSET(lfds[i], &rfds)){ if((rsiz = read(lfds[i], buf, SOCKBUFSIZ)) < 1){ lfds[i] = -1; } else { cbdatumcat(each[i], buf, rsiz); } } } } data = cblistopen(); for(i = 0; i < fdn; i++){ cblistpush(data, cbdatumptr(each[i]), cbdatumsize(each[i])); } free(lfds); for(i = 0; i < fdn; i++){ cbdatumclose(each[i]); } free(each); return data; } /* get the address of a hostname */ const char *mygethostaddr(const char *host){ static char buf2[ESTPATHBUFSIZ]; struct hostent *info; char *buf1; if(!(info = gethostbyname(host)) || !info->h_addr_list[0]) return NULL; buf1 = inet_ntoa(*(struct in_addr *)info->h_addr_list[0]); sprintf(buf2, "%s", buf1); return buf2; } /* get the client socket of an address and a port */ int mygettcpconn(const char *addr, unsigned short port){ int sockfd; struct sockaddr_in address; struct linger li; memset(&address, 0, sizeof(address)); address.sin_family = AF_INET; if(!inet_aton(addr, &address.sin_addr)) return -1; address.sin_port = htons(port); if((sockfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) == -1) return -1; li.l_onoff = 1; li.l_linger = 100; if(setsockopt(sockfd, SOL_SOCKET, SO_LINGER, (char *)&li, sizeof(li)) == -1){ close(sockfd); return -1; } if(connect(sockfd, (struct sockaddr *)&address, sizeof(address)) == -1){ close(sockfd); return -1; } return sockfd; } /* Write into a socket. */ int sockwrite(int fd, const void *buf, int size){ const char *lbuf; int rv, wb; assert(fd >= 0 && buf && size >= 0); lbuf = buf; rv = 0; do { wb = write(fd, lbuf, size); switch(wb){ case -1: if(errno != EINTR) return -1; case 0: break; default: lbuf += wb; size -= wb; rv += wb; break; } } while(size > 0); return rv; } /* Extract information from XHTML */ void extractinfo(const char *body, METAWORD *words, int *wnp, int wmax, METADOC *docs, int *dnp, int dmax){ CBLIST *elems; CBMAP *attrs; CBDATUM *summary; const char *elem, *ename, *cname, *rname, *id, *tmp, *last, *label; const char *uri, *title, *author, *date, *type, *enc, *size, *rsc, *word; char *pesc, *wraw, *wesc, numbuf[ESTNUMBUFSIZ]; int i, j, tid, did, score, sumnum, inspan; elems = cbxmlbreak(body, TRUE); tid = -1; did = -1; score = -1; uri = NULL; label = NULL; title = NULL; author = NULL; date = NULL; type = NULL; enc = NULL; size = NULL; rsc = NULL; sumnum = -1; word = NULL; for(i = 0; i < cblistnum(elems); i++){ elem = cblistval(elems, i, NULL); if(elem[0] == '<'){ if(elem[1] == '/'){ if(!strcmp(elem, "")){ if(uri && uri[0] != '<' && *dnp < dmax){ docs[*dnp].score = score; docs[*dnp].uri = cbmemdup(uri, -1); docs[*dnp].label = cbmemdup(label && label[0] != '<' ? label : "", -1); docs[*dnp].title = cbmemdup(title && title[0] != '<' ? title : "", -1); docs[*dnp].author = cbmemdup(author && author[0] != '<' ? author : "", -1); docs[*dnp].date = cbmemdup(date && date[0] != '<' ? date : "", -1); docs[*dnp].type = cbmemdup(type && type[0] != '<' ? type : "", -1); docs[*dnp].enc = cbmemdup(enc && enc[0] != '<' ? enc : "", -1); docs[*dnp].size = cbmemdup(size && size[0] != '<' ? size : "", -1); docs[*dnp].relsc = cbmemdup(rsc && rsc[0] != '<' ? rsc : "", -1); summary = cbdatumopen("", 0); inspan = FALSE; last = NULL; for(j = sumnum + 1; j < cblistnum(elems); j++){ tmp = cblistval(elems, j, NULL); if(tmp[0] == '<'){ if(!strcmp(tmp, "")) break; if(j >= 2 && inspan && last && !strcmp(tmp, "")){ attrs = cbxmlattrs(cblistval(elems, j - 2, NULL)); ename = cbmapget(attrs, "", 0, NULL); rname = cbmapget(attrs, "rel", -1, NULL); if(ename && !strcmp(ename, "a") && rname && !strcmp(rname, "narrow")){ pesc = cburlencode(phrase, -1); wraw = cbxmlunescape(last); wesc = cburlencode(wraw, -1); cbdatumcat(summary, "", -1); cbdatumcat(summary, "", -1); cbdatumcat(summary, last, -1); cbdatumcat(summary, "", -1); cbdatumcat(summary, "", -1); free(wesc); free(wraw); free(pesc); } cbmapclose(attrs); continue; } if(!strcmp(tmp, "")){ inspan = FALSE; last = NULL; continue; } attrs = cbxmlattrs(tmp); ename = cbmapget(attrs, "", -1, NULL); if(ename && !strcmp(ename, "span")){ inspan = TRUE; cbmapclose(attrs); continue; } cbmapclose(attrs); } else { last = tmp; } if(!inspan) cbdatumcat(summary, tmp, -1); } docs[*dnp].summary = cbdatumtomalloc(summary, NULL); (*dnp)++; } tid = -1; did = -1; score = -1; uri = NULL; title = NULL; author = NULL; date = NULL; type = NULL; enc = NULL; size = NULL; rsc = NULL; sumnum = -1; } } else if(elem[1] != '?' && elem[1] != '!'){ attrs = cbxmlattrs(elem); if(!(ename = cbmapget(attrs, "", 0, NULL))) ename = ""; if(!(cname = cbmapget(attrs, "class", -1, NULL))) cname = ""; if(!(id = cbmapget(attrs, "id", -1, NULL))) id = ""; if(!strcmp(ename, "span") || !strcmp(ename, "a") || !strcmp(ename, "em") || !strcmp(ename, "dd")){ if(!strcmp(cname, "pt")){ tmp = cblistval(elems, ++i, NULL); score = tmp ? (relsc[0] != '\0' ? atof(tmp) * 100 : atoi(tmp)) : 0; } else if(!strcmp(cname, "uri")){ uri = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "label")){ label = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "title")){ title = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "author")){ author = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "date")){ date = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "type")){ type = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "encoding")){ enc = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "size")){ size = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "relsc")){ rsc = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "summary")){ sumnum = i; } else if(!strcmp(cname, "word")){ word = cblistval(elems, ++i, NULL); } else if(!strcmp(cname, "hit") && label && word && *wnp < wmax){ tmp = cblistval(elems, ++i, NULL); if(tmp && tmp[0] != '<'){ words[*wnp].hit = atoi(tmp); words[*wnp].label = cbxmlunescape(label); words[*wnp].word = cbxmlunescape(word); (*wnp)++; } word = NULL; } else if(!strcmp(id, "all") && label && *wnp < wmax){ tmp = cblistval(elems, ++i, NULL); if(tmp && tmp[0] != '<'){ words[*wnp].hit = atoi(tmp); words[*wnp].label = cbxmlunescape(label); words[*wnp].word = NULL; (*wnp)++; } } } cbmapclose(attrs); } } } cblistclose(elems); } /* compare two documents by score or date */ int mdoccompare(const void *a, const void *b){ METADOC *ap, *bp; int rv; ap = (METADOC *)a; bp = (METADOC *)b; if((rv = bp->score - ap->score) != 0) return rv; return strcmp(ap->uri, bp->uri); } /* compare two documents by reverse score or reverse date */ int mdocrcompare(const void *a, const void *b){ METADOC *ap, *bp; int rv; ap = (METADOC *)a; bp = (METADOC *)b; if((rv = ap->score - bp->score) != 0) return rv; return strcmp(ap->uri, bp->uri); } /* show information of a document */ void showdoc(const METADOC *docp, int num){ char numbuf[ESTNUMBUFSIZ]; int top; xmlprintf("
\n"); xmlprintf("
"); xmlprintf("%d: ", num); xmlprintf("%s", docp->uri, ++tabidx, docp->title[0] != '\0' ? docp->title : "(untitled)"); if(docp->label[0] != '\0') xmlprintf(" (%s)", docp->label); if(relsc[0] != '\0'){ sprintf(numbuf, "%.2f", docp->score / 100.0); xmlprintf(" (%@%%)", numbuf); } else { xmlprintf(" (%d pt.)", docp->score); } xmlprintf("
\n"); if(docp->summary[0] != '\0'){ xmlprintf("
"); xmlprintf("%s", docp->summary); if(docp->relsc[0] != '\0'){ xmlprintf("%@", scriptname, max, drep, sort, expr, chkstr, docp->relsc, lrelated); } xmlprintf("
\n"); } xmlprintf("
"); top = TRUE; if(docp->author[0] != '\0'){ if(!top) xmlprintf(" "); xmlprintf("%s: %s", lauthor, docp->author); top = FALSE; } if(docp->date[0] != '\0'){ if(!top) xmlprintf(" "); xmlprintf("%s: %s", ldate, docp->date); top = FALSE; } if(docp->type[0] != '\0'){ if(!top) xmlprintf(" "); xmlprintf("%s: %s", ltype, docp->type); top = FALSE; } if(docp->enc[0] != '\0'){ if(!top) xmlprintf(" "); xmlprintf("%s: %s", lenc, docp->enc); top = FALSE; } if(docp->size[0] != '\0'){ if(!top) xmlprintf(" "); xmlprintf("%s: %s", lsize, docp->size); top = FALSE; } xmlprintf("
\n"); xmlprintf("
"); xmlprintf("%s: %s", luri, docp->uri, docp->uri); xmlprintf("
\n"); if(showsc && docp->relsc[0] != '\0') xmlprintf("
%@
\n", docp->relsc); xmlprintf("
\n"); } /* show version information */ void showversion(void){ xmlprintf("Powered by Estraier version %@.\n", _EST_VERSION); } /* output log */ void outputlog(void){ FILE *ofp; const char *claddr; char date[ESTDATEBUFSIZ]; time_t t; struct tm *tp; if(logfile[0] == '\0') return; if(!(ofp = fopen(logfile, "ab"))) return; if(!(claddr = getenv("REMOTE_ADDR"))) claddr = "0.0.0.0"; sprintf(date, "0000/00/00 00:00:00"); if((t = time(NULL)) != -1 && (tp = localtime(&t)) != NULL){ sprintf(date, "%04d/%02d/%02d %02d:%02d:%02d", tp->tm_year + 1900, tp->tm_mon + 1, tp->tm_mday, tp->tm_hour, tp->tm_min, tp->tm_sec); } fprintf(ofp, "%s\t%s\t%d\t%d\t%s\t%s\t%d\n", claddr, phrase, max, drep, sort, expr, page); fclose(ofp); } /* END OF FILE */