/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <poliqarp-config.h>

#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sakura/common/bs-file-backend.h>

#include "morph-parser.h"
#include "tag-lookup.h"
#include "utils.h"

static void chunk_push(struct morph_parser *this, const char *name)
{
   struct binary_pool_node *node;
   struct chunk_info *info;

   node = binary_pool_add(&this->subchunk_pool, name, strlen(name) + 1);
   info = this->chunk_info + node->id;
   if (info->stream == NULL) {
      info->pathname = string_aformat(".partial.%s." CORPUS_MARKER ".subchunk.item.%s",
         this->base_name, name);
      info->stream = xfopen(info->pathname, true);
      info->name = node->data;
   }
   fwrite(&this->pos_index, sizeof this->pos_index, 1, info->stream);
   this->chunk_stack[this->chunk_index++] = node->id;
}

static void chunk_pop(struct morph_parser *this)
{
   struct chunk_info *info;

   info = this->chunk_info + this->chunk_stack[--this->chunk_index];
   assert(info->stream != NULL);
   fwrite(&this->pos_index, sizeof this->pos_index, 1, info->stream);
}

static void chunk_init(struct morph_parser *this)
{
   this->chunk_index = 0;
   this->pos_index = 0;
}

static void chunk_close_all(struct morph_parser *this)
{
   size_t i;
   for (i = 0; i < this->subchunk_pool.num_items; ++i)
      if (this->chunk_info[i].stream) {
         fclose(this->chunk_info[i].stream);
         this->chunk_info[i].stream = NULL;
      }
}

static void chunk_unlink_all(struct morph_parser *this)
{
   size_t i;
   for (i = 0; i < this->subchunk_pool.num_items; ++i)
      unlink(this->chunk_info[i].pathname);
}

#ifndef HAVE_STRNDUP
static char *strndup(const char *s, size_t n)
{
   char *result;
   size_t len = strlen(s);

   if (len > n)
      len = n;
   result = malloc(len + 1);
   if (result == NULL)
      return NULL;
   memcpy(result, s, len);
   result[len] = '\0';
   return result;
}
#endif

static void process_groups_start(struct morph_parser *this, const char *id)
{
   list_t l, list = (list_t)hash_table_get(&this->syntax->htfrom, id);
   for (l = list; l; l = l->next) {
      struct group *oldgroup = (struct group *)l->item;
      struct group_info *group = this->groups_ready + this->groups_size;
      group->from = this->pos_index;
      group->to = group->synh = group->semh = UNKNOWN;
      group->id = oldgroup->id;
      group->type = binary_pool_add_string_lowercase(&this->group_type_pool,
         oldgroup->type)->id;
      /* We leave group->length unfilled, it will be dealt with later. */

      this->nest++;
      if (++this->groups_size == this->groups_max) {
         this->groups_max *= 2;
         this->groups_ready = realloc(this->groups_ready, 
            this->groups_max * sizeof(struct group_info));
      }
   }
}

static void process_groups_head(struct morph_parser *this, const char *id)
{
   int i;
   for (i = 0; i < this->groups_size; i++) {
      struct group_info *group = (struct group_info *)this->groups_ready + i;
      list_t lst, descs = (list_t)hash_table_get(&this->syntax->htsynh, id);
      for (lst = descs; lst; lst = lst->next) {
         struct group *oldgroup = (struct group *)lst->item;
         if (group->id == oldgroup->id) {
            group->synh = this->pos_index;
            break;
         }
      }
      descs = (list_t)hash_table_get(&this->syntax->htsemh, id);
      for (lst = descs; lst; lst = lst->next) {
         struct group *oldgroup = (struct group *)lst->item;
         if (group->id == oldgroup->id) {
            group->semh = this->pos_index;
            break;
         }
      }
   }
}

static int compare_groups(const void *g1, const void *g2)
{
   const struct group_info *group1 = (const struct group_info *)g1,
                           *group2 = (const struct group_info *)g2;
   return group1->from < group2->from ? -1 :
          group1->from > group2->from ? 1 : 
          group1->to > group2->to ? -1 :
          group1->to < group2->to ? 1 : 
          0;
}

static void process_groups_end(struct morph_parser *this, const char *id)
{
   int i;
   for (i = 0; i < this->groups_size; i++) {
      struct group_info *group = (struct group_info *)this->groups_ready + i;
      list_t lst, descs = (list_t)hash_table_get(&this->syntax->htto, id);
      for (lst = descs; lst; lst = lst->next) {
         struct group *oldgroup = (struct group *)lst->item;
         if (group->id == oldgroup->id) {
            group->to = this->pos_index;
            this->nest--;
            break;
         }
      }
   }
}
   
static void dump_group(struct group_info *group, struct morph_parser *this)
{
   this->syntax_bs.add_bit(this->syntax_bs.data, this->synchro ? 1 : 0);
   if (this->synchro) 
      encode_binary(&this->syntax_bs, group->from, 32);
   else
      encode_gamma(&this->syntax_bs, group->from - this->last_from + 1);
   encode_gamma(&this->syntax_bs, group->to - group->from + 1);
   encode_gamma(&this->syntax_bs, group->type + 1);
   if (group->type == 0) 
      encode_gamma(&this->syntax_bs, group->length);
   else {
      int len = clog2(group->to - group->from + 1);
      this->syntax_bs.add_bit(this->syntax_bs.data, 
         group->synh == UNKNOWN ? 0 : 1);
      this->syntax_bs.add_bit(this->syntax_bs.data, 
         group->semh == UNKNOWN ? 0 : 1);
      if (group->synh != UNKNOWN)
         encode_binary(&this->syntax_bs, group->synh - group->from, len);
      if (group->semh != UNKNOWN)
         encode_binary(&this->syntax_bs, group->semh - group->from, len);
   }
   this->last_from = group->from;
   this->last_to = group->to;
   if (this->max_to < group->to)
      this->max_to = group->to;
   this->synchro = false;
}        

static void dump_group_partial(struct group_info *group, struct morph_parser *parser)
{
   fwrite(group, sizeof(*group), 1, parser->syntax_stream);
}

static void calculate_coord_lengths(struct morph_parser *this)
{
   int i, j;
   for (i = 0; i < this->groups_size; i++) {
      if (this->groups_ready[i].type == 0) {
         j = i;
         while (j < this->groups_size && 
                this->groups_ready[j].to <= this->groups_ready[i].to)
         {
            j++;
         }
         this->groups_ready[i].length = j - i;
      }
   }
}

static void dump_ready_groups(struct morph_parser *this)
{
   if (this->nest == 0) {
      int i;

      qsort(this->groups_ready, this->groups_size, 
         sizeof(struct group_info), compare_groups);
      calculate_coord_lengths(this);
      for (i = 0; i < this->groups_size; i++)
         dump_group_partial(this->groups_ready + i, this);
      this->groups_size = 0;
   }
}

static void process_groups(struct morph_parser *this, const char *id)
{
   process_groups_start(this, id);
   process_groups_head(this, id);
   process_groups_end(this, id);
   dump_ready_groups(this);
}

static void start_tag_handler(void *p, const xmlChar *xml_name, const xmlChar **xml_attr)
{
   const char *name = (const char*)xml_name;
   const char **attr = (const char**)xml_attr;
   struct morph_parser *this = p;
   switch (lookup_tag(name)) {
      case TOK_TOK:
         this->disamb_index = 0;
         this->amb_index = 0;
         this->broken_tok = false;
         /* the order of the following two statements is correct */
         this->pos.orth_space_id = this->space;
         this->space = true;
         if (this->syntax->have_syntax && attr && *attr && 
             strcmp(*attr, "id") == 0) 
         {
            process_groups(this, attr[1]);
         }
         break;
      case TOK_LEX:
         this->broken_lex = false;
         this->have_disamb = (attr && attr[0] &&
            strcmp(attr[0], "disamb") == 0 && 
            attr[1] && strcmp(attr[1], "1") == 0) ? true : false;
         break;
      case TOK_ORTH:
      case TOK_CTAG:
      case TOK_BASE:
         this->accumulate = true;
         string_purge(this->concat_buffer);
         break;

      case TOK_NS:
         this->space = false;
         break;

      case TOK_UNKNOWN:
         vp_eprintf(_("%s/%s:%d: unknown tag %s\n"), this->dirname, 
            this->filename, this->xml_parser->input->line, name);
         this->num_errors++;
         break;

      case TOK_CHUNK:
         if (attr && attr[0]) {
            const char *ptr;
            char *copy;
            for (ptr = attr[1]; *ptr && !ascii_isdigit(*ptr); ++ptr);
            copy = strndup(attr[1], ptr - attr[1]);
            chunk_push(this, copy);
            free(copy);
         }
         break;

      default:
         break;
   }
}

/* --------------------------------------------------------------- */

static int compare_interp(const void *p1, const void *p2)
{
   const struct poliqarp_binary_interp *i1 = p1;
   const struct poliqarp_binary_interp *i2 = p2;
   return memcmp(i1, i2, sizeof *i1);
}

static void end_tag_handler(void *p, const xmlChar *xml_name)
{
   struct morph_parser *this = p;
   struct binary_pool_node *node;
   const char *name = (const char*)xml_name;

   switch (lookup_tag(name)) {
      case TOK_TOK:
         if (this->broken_tok == false) {
            qsort(this->disamb, this->disamb_index, sizeof *this->disamb, 
               compare_interp);
            /* one ... */
            node = binary_pool_add(&this->interp_disamb_pool, this->disamb, 
               sizeof *this->disamb * this->disamb_index);
            assert(node != NULL);
            this->pos.interp_disamb_id = node->id;
            /* ...and two */
            node = binary_pool_add(&this->interp_amb_pool, this->amb, 
               sizeof *this->amb * this->amb_index);
            assert(node != NULL);
            this->pos.interp_amb_id = node->id;
            /* write me */
            fwrite(&this->pos, sizeof this->pos, 1, this->corpus_stream);
            /* increment counter */
            this->pos_index++;
         }
         break;

      case TOK_LEX:
         if (!this->broken_lex) {
            if (this->have_disamb) 
               this->disamb_index++;
            this->amb_index++;
         }
         break;

      case TOK_ORTH:
         if (string_len(this->concat_buffer) != 0) {
            node = binary_pool_add(&this->orth_pool, 
               string_str(this->concat_buffer), 
               string_len(this->concat_buffer) + 1);
            assert(node != NULL);
            this->pos.orth_space_id |= node->id << 1;
         } else {
            vp_eprintf(_("%s/%s:%d: tag %s is empty\n"), this->dirname, 
               this->filename, this->xml_parser->input->line,
               name);
            this->broken_tok = true;
            this->num_errors++;
         }
         this->accumulate = false;
         break;

      case TOK_CTAG:
         if (string_len(this->concat_buffer) != 0) {
            node = binary_pool_add(&this->tag_pool, 
               string_str(this->concat_buffer), 
               string_len(this->concat_buffer) + 1);
            assert(node != NULL);
            if (this->have_disamb)
               this->disamb[this->disamb_index].tag_id = node->id;
            this->amb[this->amb_index].tag_id = node->id;
         } else {
            vp_eprintf(_("%s/%s:%d: tag %s is empty\n"), this->dirname, 
               this->filename, this->xml_parser->input->line,
               name);
            this->broken_lex = true;
            this->num_errors++;
         }
         this->accumulate = false;
         break;

      case TOK_BASE:
         if (string_len(this->concat_buffer) != 0) {
            if (this->have_disamb) {
               node = binary_pool_add(&this->base_disamb_pool, 
                  string_str(this->concat_buffer), 
                  string_len(this->concat_buffer) + 1);
               assert(node != NULL);
               this->disamb[this->disamb_index].base_id = node->id;
            }
            node = binary_pool_add(&this->base_amb_pool, 
               string_str(this->concat_buffer), 
               string_len(this->concat_buffer) + 1);
            assert(node != NULL);
            this->amb[this->amb_index].base_id = node->id;
         } else {
            vp_eprintf(_("%s/%s:%d: tag %s is empty\n"), this->dirname, 
               this->filename, this->xml_parser->input->line,
               name);
            this->broken_lex = true;
            this->num_errors++;
         }
         /* this->store_to_buffer = false; */
         this->accumulate = false;
         break;

      case TOK_CHUNK:
         chunk_pop(this);
         break;

      default:
         break;
   }
}

static void character_data_handler(void *p, const xmlChar *s, int len)
{
   struct morph_parser *this = p;
   if (this->accumulate)
      string_append_buf(this->concat_buffer, (const char*)s, len);
}

static void morph_parser_destroy(struct bp_parser *base)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);
   struct file_obs *bs = (struct file_obs *)(this->syntax_bs.data);
   uint8_t size;
   bool need_syntax;

   string_free(this->concat_buffer);
   binary_pool_free(&this->orth_pool);
   binary_pool_free(&this->base_disamb_pool);
   binary_pool_free(&this->base_amb_pool);
   binary_pool_free(&this->interp_disamb_pool);
   binary_pool_free(&this->interp_amb_pool);
   binary_pool_free(&this->tag_pool);
   binary_pool_free(&this->subchunk_pool);
   xmlFreeParserCtxt(this->xml_parser);
   fclose(this->corpus_stream);
   size = bs->bits_used % 8;

   file_obs_flush_partial(bs);
   need_syntax = (ftello(bs->file) != 0);
   /* This enables us to know exactly how many meaningful bits there are
    * in the on-disk bit stream. */
   if (need_syntax)
      fwrite(&size, 1, 1, bs->file);
   fclose(this->syntax_stream);
   fclose(this->syntax_sync_stream);
   if (!need_syntax) {
      /* no syntax data generated, remove unnecessary files */
      char *path;
      path = string_aformat("%s.poliqarp.syntax.image", this->base_name); unlink(path); free(path);
      path = string_aformat("%s.poliqarp.syntax.offset", this->base_name); unlink(path); free(path);
      path = string_aformat("%s.poliqarp.grtype.image", this->base_name); unlink(path); free(path);
      path = string_aformat("%s.poliqarp.grtype.offset", this->base_name); unlink(path); free(path);
   }
   marena_destroy(&this->arena);
   chunk_close_all(this);
   free(base);
}

static void morph_parser_stats(struct bp_parser *base)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);

   vp_printf(_("Hash table statistics\n"));
   vp_nest(vp_out);
   vp_printf(_("Orth pool statistics\n"));
   binary_pool_debug(&this->orth_pool, vp_out);
   vp_printf(_("Base (disambiguated) pool statistics\n"));
   binary_pool_debug(&this->base_disamb_pool, vp_out);
   vp_printf(_("Base (ambiguous) pool statistics\n"));
   binary_pool_debug(&this->base_amb_pool, vp_out);
   vp_printf(_("Tag pool statistics\n"));
   binary_pool_debug(&this->tag_pool, vp_out);
   vp_printf(_("Interpretation (disambiguated) pool statistics\n"));
   binary_pool_debug(&this->interp_disamb_pool, vp_out);
   vp_printf(_("Interpretation (all) pool statistics\n"));
   binary_pool_debug(&this->interp_amb_pool, vp_out);
   vp_printf(_("Subchunk pool statistics\n"));
   binary_pool_debug(&this->subchunk_pool, vp_out);
   vp_unnest(vp_out);
}

static void output_err_group_part(string_t s, uint32_t part)
{
   if (part == UNKNOWN)
      string_append_str(s, "UNKNOWN");
   else
      string_append_long(s, (long)part);
}

static void output_err_group(morph_parser *parser, const struct group_info *group)
{
   string_t s = string_init("   from=");
   output_err_group_part(s, group->from);

   string_append_str(s, " to=");
   output_err_group_part(s, group->to);

   string_append_str(s, " synh=");
   output_err_group_part(s, group->synh);

   string_append_str(s, " semh=");
   output_err_group_part(s, group->semh);

   string_append_str(s, " type=");
   string_append_str(s, (const char *)binary_pool_get_by_id(
         &parser->group_type_pool, group->type));

   vp_eprintf("%s\n", string_str(s));
   string_free(s);
} 

static void morph_parser_work(struct bp_parser_env *env, struct bp_parser *base,
   const char *pathname, const char *target_dir, const char *target_file)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);
   char *partial_path, *syntax_path;

   /* generate partial file names */
   partial_path = string_aformat(".partial.%s."CORPUS_MARKER".corpus.image", this->base_name);
   syntax_path = string_aformat(".partial.%s.poliqarp.syntax.image", this->base_name);

   if (base->incremental && 
       check_partial_file(partial_path, target_file) == PARTIAL_FILE_OK)
   {
      free(syntax_path);
      free(partial_path);
      return;
   }
   
   /* open output streams */
   this->corpus_stream = xfopen(partial_path, true);
   this->syntax_stream = xfopen(syntax_path, true);

   /* set data for SAX handlers */
   this->dirname = target_dir;
   this->filename = target_file;
   this->groups_max = 1; /* FIXME: initialize this to some sane value, say, 32 */
   this->groups_size = this->nest = 0;
   this->groups_ready = malloc(this->groups_max * sizeof(*this->groups_ready));

   /* initialize chunks data */
   chunk_init(this);

   if (generic_parse_loop(target_dir, target_file, this->xml_parser) == -1) {
      vp_eprintf(_("There were some errors\n"));
      vp_eprintf(_("Removing partial files from %s\n"), target_dir);
      chunk_unlink_all(this);
      unlink(partial_path);
      unlink(syntax_path);
      this->num_errors++;
   }

   if (this->nest > 0) {
      int i = 0;

      vp_eprintf("Warning: Incorrect nesting detected in %s%s\n",
         target_dir, target_file);
      vp_eprintf("Syntax data for this document will be incomplete\n");
      vp_eprintf("The following groups have undefined parts:\n");
      
      for (; i < this->groups_size; i++) {
         struct group_info *group = (struct group_info *)this->groups_ready + i;
         if (group->from == UNKNOWN ||
             group->to == UNKNOWN   ||
             (group->type != 0 && group->type != 2 && (group->synh == UNKNOWN || group->semh == UNKNOWN)))
         {
            output_err_group(this, group);
         }
      }
      
      env->num_errors++;
   }
   /* close all chunks */
   chunk_close_all(this);
   
   free(syntax_path);
   free(partial_path); 
   fclose(this->corpus_stream);
   fclose(this->syntax_stream);
   this->filename = NULL;
   this->dirname = NULL;
}

static void morph_parser_merge(struct bp_parser_env *env, 
   struct bp_parser *base, const char *pathname, const char *target_dir,
   const char *target_file)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);
   char *path;
   size_t num_pos_read = 0, old_pos_index;
   size_t i;

   path = string_aformat(".partial.%s." CORPUS_MARKER ".corpus.image", this->base_name);
   if (access(path, R_OK | F_OK) == 0) {
      struct poliqarp_binary_segment 
         buf[4096 / sizeof(struct poliqarp_binary_segment)];
      FILE *partial = xfopen(path, false);
      FILE *whole = this->corpus_stream;
      size_t num_read = 0;

      assert(partial != NULL);
      while ((num_read = fread(buf, sizeof *buf, sizeof buf / sizeof *buf, 
         partial)) > 0) 
      {
         fwrite(buf, sizeof *buf, num_read, whole);
         num_pos_read += num_read;
      }
      fclose(partial);
   } else {
      vp_eprintf(_("Cannot find partial file in %s "
         "(probably it wasn't built correctly)\n"), pathname);
      this->num_errors++;
   }
   free(path);

   for (i = 0; i < this->subchunk_pool.num_items; ++i) {
      path = string_aformat(".partial.%s." CORPUS_MARKER ".subchunk.item.%s",
         this->base_name, this->chunk_info[i].name);
      if (access(path, R_OK | F_OK) == 0) {
         uint32_t buf[4096 / sizeof(uint32_t)];
         FILE *partial = xfopen(path, false);
         FILE *whole = this->chunk_info[i].stream;
         size_t num_read = 0;
         size_t j;
         
         assert(partial != NULL);
         while ((num_read = fread(buf, sizeof *buf, sizeof buf / sizeof *buf, 
            partial)) > 0) 
         {
            for (j = 0; j < num_read; ++j)
               buf[j] += this->pos_index;
            fwrite(buf, sizeof *buf, num_read, whole);
         }
         fclose(partial);
      }
      free(path);
   }
   
   old_pos_index = this->pos_index;
   this->pos_index += num_pos_read;
   path = string_aformat(".partial.%s.poliqarp.syntax.image", this->base_name);
   if (access(path, R_OK | F_OK) == 0) {
      struct group_info buf[1024];
      FILE *partial = xfopen(path, false);
      size_t num_read = 0;

      assert(partial != NULL);
      while ((num_read = fread(buf, sizeof *buf, sizeof buf / sizeof *buf, 
         partial)) > 0) 
      {
         size_t i;
         for (i = 0; i < num_read; i++) {
            uint32_t delta;
            buf[i].from += old_pos_index;
            buf[i].to += old_pos_index;
            if (buf[i].synh != UNKNOWN)
               buf[i].synh += old_pos_index;
            if (buf[i].semh != UNKNOWN)
               buf[i].semh += old_pos_index;
            if (this->max_to > buf[i].to)
               delta = 0;
            else
               delta = (buf[i].to / 1024) - (this->max_to / 1024);
            if (this->merge_first) {
               this->merge_first = false;
               delta++;
            }
            while (delta) {
               struct file_obs *bs = (struct file_obs *)(this->syntax_bs.data);
               uint32_t pos = bs->blocks_written * bs->block_size * 8 + bs->bits_used;
               this->synchro = true;
               fwrite(&pos, sizeof(pos), 1, this->syntax_sync_stream);
               delta--;
            }
            dump_group(buf + i, this);
         }
      }
      fclose(partial);
   } else {
      vp_eprintf(_("Cannot find partial file in %s "
         "(probably it wasn't built correctly)\n"), pathname);
      this->num_errors++;
   }
   free(path);
}

static int morph_parser_preprocess(struct bp_parser *base)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);

   bp_read_pool(base, &this->orth_pool, "orth", this->base_name);
   bp_read_pool(base, &this->base_disamb_pool, "base1", this->base_name);
   bp_read_pool(base, &this->base_amb_pool, "base2", this->base_name);
   bp_read_pool(base, &this->tag_pool, "tag", this->base_name);
   bp_read_pool(base, &this->interp_disamb_pool, "interp1", this->base_name);
   bp_read_pool(base, &this->interp_amb_pool, "interp2", this->base_name);
   bp_read_pool(base, &this->subchunk_pool, "subchunk", this->base_name);
   bp_read_pool(base, &this->group_type_pool, "grtype", this->base_name);
   return 0;
}

static int prepare_final_file(struct bp_parser *base)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);
   char *path;
   size_t i, num_chunks = this->subchunk_pool.num_items;
   struct chunk_info *info;
   struct file_obs *obs_backend;

   vp_printf(_("Merging...\n"));
   vp_nest(vp_out);
   path = string_aformat("%s." CORPUS_MARKER ".corpus.image", this->base_name);
   this->corpus_stream = xfopen(path, true);
   vp_printf(_("Merging corpus to %s\n"), path);
   free(path);
   vp_unnest(vp_out);

   vp_printf(_("Merging chunk data...\n"));
   this->pos_index = 0;
   vp_nest(vp_out);
   for (i = 0; i < num_chunks; ++i) {
      info = &this->chunk_info[i];
      path = string_aformat("%s." CORPUS_MARKER ".subchunk.item.%s",
         this->base_name, info->name);
      info->stream = xfopen(path, true);
      vp_printf(_("Merging chunk %s to %s\n"), info->name, path);
      free(path);
   }
   vp_unnest(vp_out);

   path = string_aformat("%s.poliqarp.syntax.image", this->base_name);
   this->syntax_stream = xfopen(path, true);
   free(path);
   path = string_aformat("%s.poliqarp.syntax.offset", this->base_name);
   this->syntax_sync_stream = xfopen(path, true);
   free(path);
   obs_backend = malloc(sizeof(*obs_backend));
   file_obs_create(obs_backend, 4096, this->syntax_stream);
   file_obs_set_backend(&this->syntax_bs, obs_backend);
   this->synchro = true;
   this->merge_first = true;
   this->last_from = 0;
   this->last_to = 0;
   this->max_to = 0;

   return 0;
}

static int morph_parser_postprocess(struct bp_parser *base)
{
   morph_parser *this = (void *)(((char *)base) + sizeof *base);

   bp_write_pool2(base, &this->orth_pool, "orth", this->base_name);
   bp_write_pool(base, &this->base_disamb_pool, "base1", this->base_name);
   bp_write_pool(base, &this->base_amb_pool, "base2", this->base_name);
   bp_write_pool(base, &this->interp_disamb_pool, "interp1", this->base_name);
   bp_write_pool(base, &this->interp_amb_pool, "interp2", this->base_name);
   bp_write_pool(base, &this->tag_pool, "tag", this->base_name);
   bp_write_pool(base, &this->subchunk_pool, "subchunk", this->base_name);
   bp_write_pool(base, &this->group_type_pool, "grtype", this->base_name);
   prepare_final_file(base);
   return 0;
}

/* FIXME: handle errors properly */
struct bp_parser *morph_parser_create(struct bp_parser *syntax_parser,
   const char *base_name, bool incremental)
{
   struct bp_parser *base;
   struct syntax_parser *syntax;
   morph_parser *this;

   base = malloc(sizeof *base + sizeof *this);
   if (base == NULL)
      return NULL;

   base->destroy = morph_parser_destroy;
   base->stats = morph_parser_stats;
   base->work = morph_parser_work;
   base->merge = morph_parser_merge;
   base->preprocess = morph_parser_preprocess;
   base->postprocess = morph_parser_postprocess;
   base->num_errors = 0;
   base->incremental = incremental;

   syntax = (void *)(((char *)syntax_parser) + sizeof(*syntax_parser));
   this = (void *)(((char *)base) + sizeof *base);
   this->syntax = syntax;

   memset(this->disamb, 0, sizeof this->disamb);
   memset(this->amb, 0, sizeof this->amb);
   memset(this->chunk_info, 0, sizeof this->chunk_info);
   marena_create(&this->arena);

   this->concat_buffer = string_create();
   this->accumulate = false;

   tag_lookup_init();
   binary_pool_init(&this->base_disamb_pool, &this->arena, 4096 * 64 - 1);
   binary_pool_init(&this->base_amb_pool, &this->arena, 4096 * 128 - 1);
   binary_pool_init(&this->orth_pool, &this->arena, 4096 * 128 - 1);
   binary_pool_init(&this->interp_disamb_pool, &this->arena, 4096 * 128 - 1);
   binary_pool_init(&this->interp_amb_pool, &this->arena, 4096 * 128 - 1);
   binary_pool_init(&this->subchunk_pool, &this->arena, 4096 * 1 - 1);
   binary_pool_init(&this->tag_pool, &this->arena, 4096 * 2 - 1);
   binary_pool_init(&this->group_type_pool, &this->arena, 4096 * 1 - 1);

   this->base_name = marena_strdup(&this->arena, base_name);
   this->space = false; /* no space at the beginning */
   this->num_errors = 0;
   this->corpus_stream = NULL;
   
   /* enforce order of some group types */
   binary_pool_add_string2(&this->group_type_pool, "coordination"); /* 0 */
   binary_pool_add_string2(&this->group_type_pool, "conjunction");  /* 1 */
   binary_pool_add_string2(&this->group_type_pool, "conjunct");     /* 2 */

   /* set call-back handlers */
   this->sax_handler = bp_dummy_sax_handler;
   this->sax_handler.startElement = start_tag_handler;
   this->sax_handler.endElement = end_tag_handler;
   this->sax_handler.characters = character_data_handler;
   this->xml_parser = xmlCreatePushParserCtxt(&this->sax_handler, this, NULL,
      0, NULL);
   if (this->xml_parser == NULL) {
      this->num_errors++;
      vp_eprintf(_("Couldn't allocate memory for parser"));
   }

   return base;
}
