/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <poliqarp-config.h>

#include "header-parser.h"
#include "metadata.h"
#include "foolog.h"
#include "utils.h"

lptr verify_and_count_path_list(lptr rest)
{
   size_t size;
   lptr n;
   for (n = rest, size = 0; n != Qnil; n = L_CDR(n), ++size)
      if (!L_STRINGP(L_CAR(n))) {
         foolog(MSG_FATAL, _("Path must be a string"));
         return Qnil;
      }
   return L_MAKE_INT(size);
}

const char **pack_path_list(lptr rest, size_t size)
{
   const char **result;
   lptr n;
   size_t i;

   result = malloc(sizeof *result * size);
   for (n = rest, i = 0; n != Qnil; n = L_CDR(n), ++i)
      result[i] = L_STRING_TEXT(L_CAR(n));
   return result;
}

lptr create_meta(lisp_env * env, enum poliqarp_binary_metadata_type type,
   lptr name, lptr rest)
{
   const char *s_name;
   lptr list_size;
   const char **path_array;
   header_parser *parser = lisp_get_user_data(env);
   struct list_item *item;

   if (L_STRINGP(name))
      s_name = L_STRING_TEXT(name);
   else {
      foolog(MSG_FATAL, _("Name must be a string"));
      return Qnil;
   }
   list_size = verify_and_count_path_list(rest);
   if (list_size == Qnil)
      return Qnil;
   path_array = pack_path_list(rest, L_GET_INT(list_size));
   item =
      list_create(meta_item_create(binary_pool_add(&parser->key_pool, s_name,
            strlen(s_name) + 1), type, L_GET_INT(list_size), path_array));
   free(path_array);
   parser->meta_list = list_prepend(parser->meta_list, item);
   return Qnil;
}

lptr lw_meta_date(lisp_env * env, lptr name, lptr rest)
{
   return create_meta(env, POLIQARP_METADATA_DATE, name, rest);
}

lptr lw_meta_single(lisp_env * env, lptr name, lptr rest)
{
   return create_meta(env, POLIQARP_METADATA_SINGLE, name, rest);
}

lptr lw_meta_multi(lisp_env * env, lptr name, lptr rest)
{
   return create_meta(env, POLIQARP_METADATA_MULTI, name, rest);
}

struct list_item *frozbot(lptr rest, size_t size, struct list_item *head)
{
   lptr n;
   struct list_item *item;
   struct list_item *result = NULL;

   for (n = rest; n != Qnil; n = L_CDR(n)) {
      item =
         list_find2(head, meta_item_compare_by_name, L_STRING_TEXT(L_CAR(n)));
      if (item)
         result = list_prepend(result, list_create(item->item));
   }
   return result;
}

lptr lw_meta_alias(lisp_env * env, lptr name, lptr rest)
{
   const char *s_name;
   lptr list_size;
   header_parser *parser = lisp_get_user_data(env);
   struct list_item *item;

   if (L_STRINGP(name))
      s_name = L_STRING_TEXT(name);
   else {
      vp_eprintf("Name must be a string");
      return Qnil;
   }
   list_size = verify_and_count_path_list(rest);
   if (list_size == Qnil)
      return Qnil;
   item =
      list_create(meta_alias_create(binary_pool_add(&parser->key_pool, s_name,
            strlen(s_name) + 1), frozbot(rest, L_GET_INT(list_size),
            parser->meta_list)));
   parser->alias_list = list_prepend(parser->alias_list, item);
   return Qnil;
}

void flush_metadata(struct header_parser *this)
{
   struct list_item *node, *value;
   for (node = this->meta_list; node != NULL; node = node->next) {
      struct meta_item *item = node->item;
      for (value = item->values; value != NULL; value = value->next) {
         struct poliqarp_binary_metadata meta;
         if (item->type == POLIQARP_METADATA_DATE) {
            meta.type = POLIQARP_METADATA_DATE;
            meta.key = item->key_node->id;
            meta.value_as.date = *(struct poliqarp_meta_date *)(value->item);
         } else {
            const char *val = (const char *)value->item;
            struct binary_pool_node *value_node = binary_pool_add(
               &this->value_pool, val, strlen(val) + 1);
            if (item->type == POLIQARP_METADATA_SINGLE) 
               pack_meta_single(&meta, item->key_node->id, value_node->id);
            else
               pack_meta_multi(&meta, item->key_node->id, value_node->id);
         }
         store_meta(this, &meta);
      }
      list_destroy(item->values, free);
      item->values = item->valend = NULL; /* just in case */
   }
}

void process_key_value(struct header_parser *this, const char *path, 
   const char *value)
{
   size_t i;
   struct poliqarp_binary_metadata meta;
   struct list_item *node;
   struct meta_item *item;

   for (node = this->meta_list; node != NULL; node = node->next) {
      item = node->item;
      for (i = 0; i < item->num_paths; ++i) {
         if (item->def_path != NULL) {
            if (item->path + i > item->def_path)
               continue;
            if (item->path + i == item->def_path && 
               item->type != POLIQARP_METADATA_MULTI) continue;
         }
         if (poliqarp_regexp_match(item->path + i, path)) {
            struct list_item *newelem;            
            if (item->def_path != NULL && (item->path + i < item->def_path) && 
               item->values != NULL) 
            {
               list_destroy(item->values, free);
               item->values = item->valend = NULL;
            }
            if (item->type == POLIQARP_METADATA_DATE) {
               struct poliqarp_meta_date *date;
               pack_meta_date(&meta, item->key_node->id, value);
               /* however, if this date's incorrect, then proceed as if 
                * it had not existed at all */
               if (meta.value_as.date.year == 0 && 
                  meta.value_as.date.month == 0 && 
                  meta.value_as.date.day == 0)
               {
                  continue;
               }
               date = malloc(sizeof(*date));
               *date = meta.value_as.date;
               newelem = list_create(date);
            } else
               newelem = list_create(strdup(value));
            if (item->valend) 
               list_prepend(newelem, item->valend);
            else
               item->values = newelem;
            item->valend = newelem;
            item->def_path = item->path + i;
         }
      }
   }
}

/* --------------------------------------------------------------- */

static void start_tag_handler(void *p, const xmlChar *xml_name, const xmlChar **xml_attr)
{
   const char *name = (const char*)xml_name;
   const char **attr = (const char**)xml_attr;
   struct header_parser *this = p;
   text_stack_push2(&this->text_stack, name, strlen(name), '/');
   for (; attr && *attr; attr += 2) {
      text_stack_push2(&this->text_stack, *attr, strlen(*attr), ':');
      process_key_value(this, text_stack_text(&this->text_stack), attr[1]);
      text_stack_pop(&this->text_stack);
   }
   string_clear(this->text_buf);
   this->accumulate = true;
}

/* --------------------------------------------------------------- */

static void end_tag_handler(void *p, const xmlChar *name)
{
   struct header_parser *this = p;
   if (this->accumulate) {
      char *trimmed = text_normalize(string_str(this->text_buf));
      if (trimmed && strlen(trimmed)) {
         process_key_value(this, text_stack_text(&this->text_stack), trimmed);
         free(trimmed);
      }
   }
   text_stack_pop(&this->text_stack);
   this->accumulate = false;
}

/* --------------------------------------------------------------- */

static void character_data_handler(void *p, const xmlChar *s, int len)
{
   struct header_parser *this = p;
   if (this->accumulate)
      string_append_buf(this->text_buf, (const char *)s, len);
}

/* --------------------------------------------------------------- */
/* --------------------------------------------------------------- */
/* --------------------------------------------------------------- */

/* FIXME: handler errors properly */
bp_parser *header_parser_create(const char *base_name, bool incremental)
{
   bp_parser *base;
   header_parser *this;


   base = malloc(sizeof *base + sizeof *this);

   base->destroy = header_parser_destroy;
   base->stats = header_parser_stats;
   base->work = header_parser_work;
   base->merge = header_parser_merge;
   base->preprocess = header_parser_pre_process;
   base->postprocess = header_parser_post_process;
   base->num_errors = 0;
   base->incremental = incremental;

   this = (void *) (((char *) base) + sizeof *base);

   this->text_buf = string_create();
   this->accumulate = false;
   text_stack_create(&this->text_stack);

   marena_create(&this->arena);
   binary_pool_init(&this->value_pool, &this->arena, 4096 * 4 - 1);
   binary_pool_init(&this->key_pool, &this->arena, 4096 * 4 - 1);

   this->base_name = marena_strdup(&this->arena, base_name);
   this->num_errors = 0;
   this->meta_list = NULL;
   this->alias_list = NULL;

   this->sax_handler = bp_dummy_sax_handler;
   this->sax_handler.startElement = start_tag_handler;
   this->sax_handler.endElement = end_tag_handler;
   this->sax_handler.characters = character_data_handler;
   this->xml_parser = xmlCreatePushParserCtxt(&this->sax_handler, this, NULL,
      0, NULL);
   if (this->xml_parser == NULL) {
      this->num_errors++;
      foolog(MSG_FATAL, _("Couldn't allocate memory for parser"));
   }

   /* lisp for dummies in 24 hours ;] */
   {
      struct lisp_env *env = make_lisp_env();
      struct lisp_stream *stream;
      char *config_name;

      lisp_set_user_data(env, this);

      L_DEF_SUBRX(env, "date", 1, 2, lw_meta_date);
      L_DEF_SUBRX(env, "single", 1, 2, lw_meta_single);
      L_DEF_SUBRX(env, "multi", 1, 2, lw_meta_multi);
      L_DEF_SUBRX(env, "alias", 1, 2, lw_meta_alias);

      config_name = string_aformat("%s.meta.lisp", this->base_name);
      stream = make_file_stream(env, config_name);
      free(config_name);
      if (stream) {
         lptr val;
         while ((val = lisp_read(env, stream)) != Qvoid)
            leval(env, val);
         stream->close(stream);
      }
      else {
         foolog(MSG_FATAL, _("Cannot open meta-lisp file: %s"),
            strerror(errno));
      }
      free_lisp_env(env);
   }
   return base;
}

/* --------------------------------------------------------------- */

void header_parser_destroy(struct bp_parser *base)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);

   string_free(this->text_buf);
   text_stack_destroy(&this->text_stack);
   list_destroy(this->meta_list, meta_item_destroy);
   list_destroy(this->alias_list, meta_alias_destroy);

   binary_pool_free(&this->value_pool);
   binary_pool_free(&this->key_pool);
   marena_destroy(&this->arena);
   xmlFreeParserCtxt(this->xml_parser);

   free(base);
}

/* --------------------------------------------------------------- */

void header_parser_stats(struct bp_parser *base)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);

   vp_printf(_("Hash table stats\n"));
   vp_nest(vp_out);
   vp_printf(_("Meta-key pool stats\n"));
   binary_pool_debug(&this->key_pool, vp_out);
   vp_printf(_("Meta-value pool stats\n"));
   binary_pool_debug(&this->value_pool, vp_out);
   vp_unnest(vp_out);
}

/* --------------------------------------------------------------- */

int prepare_final_stream(header_parser * this)
{
   char *path;

   vp_printf(_("Merging meta data...\n"));
   vp_nest(vp_out);
   {
      path =
         string_aformat("%s." CORPUS_MARKER ".meta.image", this->base_name);
      this->meta_stream = xfopen(path, true);
      foolog(MSG_INFO, _("Writing final stream to %s"), path);
      free(path);
   }
   vp_unnest(vp_out);
   return 0;
}

/* --------------------------------------------------------------- */

int header_parser_pre_process(struct bp_parser *base)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);

   bp_read_pool(base, &this->key_pool, "meta-key", this->base_name);
   bp_read_pool(base, &this->value_pool, "meta-value", this->base_name);
   return 0;
}

int header_parser_post_process(struct bp_parser *base)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);

   bp_write_pool(base, &this->key_pool, "meta-key", this->base_name);
   bp_write_pool(base, &this->value_pool, "meta-value", this->base_name);
   prepare_final_stream(this);
   return 0;
}

/* --------------------------------------------------------------- */

void
header_parser_work(bp_parser_env * env,
   struct bp_parser *base,
   const char *pathname, const char *target_dir, const char *target_file)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);

   char *partial_file;

   foolog(MSG_INFO, _("Header parser is processing file: %s"), pathname);

   /* generate partial file name */
   partial_file =
      string_aformat(".partial.%s." CORPUS_MARKER ".meta.image",
      this->base_name);

   if (base->incremental)
      /* check partial file */
      switch (check_partial_file(partial_file, target_file)) {
         case PARTIAL_FILE_OK:
            foolog(MSG_INFO, _("Reusing existing partial file"));
            return;
         case PARTIAL_FILE_NEEDS_REMAKE:
            foolog(MSG_NOTICE,
               _("Partial file is older than source file, remaking"));
            break;
         case PARTIAL_FILE_GONE:
            foolog(MSG_INFO, _("Building partial file"));
            break;
         default:
            abort();
      }
   else
      foolog(MSG_INFO, _("Building partial file"));

   /* open output stream */
   this->meta_stream = xfopen(partial_file, true);

   /* set data for SAX handlers */
   this->dirname = target_dir;
   this->filename = target_file;

   /* reset pick-list (important) */
   list_apply(this->meta_list, meta_item_reset);

   if (generic_parse_loop(target_dir, target_file, this->xml_parser) == -1) {
      this->num_errors++;
      foolog(MSG_WARNING, "There were some errors in the header file (%s)",
         pathname);
      foolog(MSG_WARNING, "Unlinking partial file due to errors (%s/%s)",
         target_dir, partial_file);
      unlink(partial_file);
   }

   /* cleanup */

   /* TODO: merge this with reset_picklist */
   /* do as the name says */

   free(partial_file);
   flush_metadata(this);
   list_apply2(this->meta_list, meta_item_store_undefined_value, this);
   /* list_apply2 (this->alias_list, meta_alias_reslove_and_store, this); */
   if (this->meta_stream)
      fclose(this->meta_stream);

   this->filename = NULL;
}

/* --------------------------------------------------------------- */

void
header_parser_merge(bp_parser_env * env,
   struct bp_parser *base,
   const char *pathname, const char *target_dir, const char *target_file)
{
   header_parser *this = (void *) (((char *) base) + sizeof *base);
   char *path;

   path =
      string_aformat(".partial.%s." CORPUS_MARKER ".meta.image",
      this->base_name);
   if (access(path, R_OK | F_OK) == 0) {
      char buf[4096];
      FILE *partial = xfopen(path, false);
      FILE *whole = this->meta_stream;
      size_t num_read = 0;
      while ((num_read = fread(buf, 1, sizeof buf, partial)) > 0)
         fwrite(buf, 1, num_read, whole);
      fclose(partial);
   }
   else {
      foolog(MSG_ERROR, _("Cannot find partial file in %s\n"), target_dir);
      this->num_errors++;
   }
   free(path);
}


