/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <poliqarp-config.h>

#include <getopt.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>

#include <foostring/foostring.h>

#include <sakura/common/bitstream.h>
#include <sakura/common/bs-file-backend.h>
#include <sakura/abi.h>
#include <sakura/backend-corpus.h>
#include <sakura/backend-interp.h>
#include <sakura/backend-index.h>
#include <sakura/cdf.h>

#include <bp/common.h>
#include <bp/io.h>

#include "block-list.h"
#include "bst.h"
#include "cmdline.h"
#include "heap.h"
#include "merge-output.h"
#include "occurrences.h"
#include "runs.h"
#include "vector.h"

void open_files(FILE **infile)
{
   string_t path = string_create();
   string_format(path, POLIQARP_CORPUS_IMAGE_FORMAT,
      cmdline.input_file);
   *infile = bp_open(path, "rb");
   string_free(path);
}

void open_outfile(FILE **outfile, const char *name_format)
{
   string_t path = string_create();
   string_format(path, name_format, cmdline.output_file);
   *outfile = bp_open(path, "w+b");
   string_free(path);
}

struct index {
   int type;
   struct occurrences *occurrences;
   struct poliqarp_backend_interp *interp;
   FILE *outfile;
   struct bst *bst;
};

void add_orth(struct index *index, struct poliqarp_binary_segment *segment)
{
   bst_insert(index->bst, segment->orth_space_id >> 1);
}

void add_disamb(struct index *index, struct poliqarp_binary_segment *segment)
{
   bst_insert(index->bst, segment->interp_disamb_id);
}

void add_amb(struct index *index, struct poliqarp_binary_segment *segment)
{
   bst_insert(index->bst, segment->interp_amb_id);
}

struct indices {
   struct index *indices;
   size_t size;
};

struct index_description {
   int specifier;
   const char *name_format;
   const char *description;
   void (*add)(struct index *, struct poliqarp_binary_segment *);
};

const struct index_description descriptions[] = {
   { POLIQARP_INDEX_ORTH, POLIQARP_CORPUS_ORTH_INDEX_FORMAT, 
      "index of orthographic forms", 
      add_orth },
   { POLIQARP_INDEX_DISAMB, POLIQARP_CORPUS_DISAMB_INDEX_FORMAT, 
      "index of disambiguated interpretations", 
      add_disamb },
   { POLIQARP_INDEX_AMB, POLIQARP_CORPUS_AMB_INDEX_FORMAT, 
      "index of ambiguous interpretations", 
      add_amb }
};

#define NUM_INDICES     (sizeof(descriptions) / sizeof(descriptions[0]))

struct indices create_indices()
{
   size_t i, j;
   struct indices res;
   struct poliqarp_backend_interp *interp = bp_malloc(sizeof *interp);
   struct poliqarp_error error = poliqarp_error_none;
   int rc;

   rc = poliqarp_backend_interp_open(interp, cmdline.input_file, &error);
   if (rc != 0)
      bp_fatal(poliqarp_error_message_get(&error));
  
   res.size = 0;
   for (i = 0; i < NUM_INDICES; i++)
      if (cmdline.indices & (1 << i))
         res.size++;
   assert(res.size);
   res.indices = bp_malloc(res.size * sizeof(struct index));
   j = 0;
   for (i = 0; i < NUM_INDICES; i++) {
      if (cmdline.indices & descriptions[i].specifier) {
         struct index *index = res.indices + j++;
         open_outfile(&index->outfile, descriptions[i].name_format);
         index->type = i;
         index->interp = interp;
         index->occurrences = bp_malloc(sizeof(struct occurrences));
         occurrences_create(index->occurrences, 
            cmdline.memory / (res.size * sizeof(struct occurrence)), 
            index->outfile);
         index->bst = bp_malloc(sizeof(struct bst));
         bst_create(index->bst, NULL);
      }
   }
   return res;
} 

struct indices process_segments(FILE *infile)
{
   void *buffer = bp_malloc(cmdline.granularity * sizeof(struct poliqarp_binary_segment));
   struct indices indices = create_indices();
   size_t j, pos, corpus_size, prog = 0, prog_hlp = 0;
   uint32_t document;

   corpus_size = bp_file_size(infile) / sizeof(struct poliqarp_binary_segment);

   bp_printf("Stage 0: Creating temporary file from %" PRIuSIZE " segments\n", corpus_size);
   bp_show_progress(0);

   for (pos = 0, document = 0; pos < corpus_size; document++) {
      size_t i, j, num_run = corpus_size - pos;
      struct poliqarp_binary_segment *segment =
         (struct poliqarp_binary_segment *)buffer;

      for (j = 0; j < indices.size; j++) 
         bst_empty(indices.indices[j].bst);
      if (num_run > cmdline.granularity)
         num_run = cmdline.granularity;
      fread(buffer, sizeof(struct poliqarp_binary_segment), num_run, infile);
      for (i = 0; i < num_run; i++) {
         POLIQARP_SEGMENT_LE_TO_HE(*segment);
         pos++;
         for (j = 0; j < indices.size; j++) 
            descriptions[indices.indices[j].type].add(indices.indices + j, segment);
         segment++;
         prog_hlp += 100;
         if (prog_hlp >= corpus_size) {
            prog_hlp -= corpus_size;
            bp_show_progress(++prog);
         }
      }
      for (j = 0; j < indices.size; j++) 
         occurrences_dump_segments(indices.indices[j].occurrences, document, 
            indices.indices[j].bst->root);
   }
   bp_printf("\n\n");

   for (j = 0; j < indices.size; j++) {
      occurrences_flush(indices.indices[j].occurrences);
      indices.indices[j].occurrences->num_documents = document;
   }

   bp_close(infile);
   free(buffer);
   for (j = 0; j < indices.size; j++) {
      bst_destroy(indices.indices[j].bst);
      free(indices.indices[j].bst);
   }
   poliqarp_backend_interp_close(indices.indices[0].interp);
   free(indices.indices[0].interp);

   return indices;
}

off64_t merge_and_recode(FILE *outfile, struct index *index,
   struct vector **map)
{
   struct free_block_list block_list;
   struct runs runs;
   struct heap heap;
   struct merge_output output;
   struct occurrences *occurrences = index->occurrences;
   struct vector *block_map = bp_malloc(sizeof *block_map);
   size_t prog = 0, prog_hlp = 0, occnum;
   off64_t numbytes;
   FILE *offsets;
   char *fmt;

   fmt = string_aformat("%s.offset", descriptions[index->type].name_format);
   open_outfile(&offsets, fmt);
   free(fmt);

   vector_create(block_map, VECTOR_DEFAULT);
   free_block_list_create(&block_list, outfile, 
      occurrences->bs_backend.block_size, block_map);
   runs_create(&runs, occurrences, &block_list);
   heap_create(&heap, &runs);
   merge_output_create(&output, occurrences, &block_list, offsets);
   occnum = (runs.size - 1) * occurrences->size + occurrences->remaining;
   occurrences_destroy(occurrences);
   free(occurrences);

   bp_printf("Stage 1: Merging and recompressing %" PRIuSIZE " occurrences\n", occnum);
   bp_show_progress(0);
   while (!heap_empty(&heap)) {
      int i = heap_extract_min(&heap);
      merge_output_occurrence(&output, runs_get_current(&runs, i));
      if (runs_fetch_occurrence(&runs, i) == 0)
         heap_insert(&heap, i);
      prog_hlp += 100;
      if (prog_hlp >= occnum) {
         prog_hlp -= occnum;
         bp_show_progress(++prog);
      }
   }

   numbytes = merge_output_get_bytes_count(&output);
   merge_output_flush(&output);

   merge_output_destroy(&output);
   heap_destroy(&heap);
   runs_destroy(&runs);
   free_block_list_destroy(&block_list);
   *map = block_map;
   bp_close(offsets);
   bp_printf("\n\n");
   return numbytes;
}

void permute(FILE *file, size_t block_size, struct vector *block_map)
{
   size_t num_blocks;
   off64_t *inv;
   void *tmp1 = bp_malloc(block_size), *tmp2 = bp_malloc(block_size);
   off64_t i, j;

   bp_printf("Stage 2: Reordering blocks\n\n");
   fseeko(file, 0, SEEK_END);
   num_blocks = ftello(file) / block_size;
   inv = bp_malloc(num_blocks * sizeof *inv);
   for (i = 0; i < num_blocks; i++)
      inv[i] = -1;
   for (i = 0; i < vector_length(block_map); i++)
      inv[vector_get(block_map, i)] = i;
   j = vector_length(block_map);
   for (i = 0; i < num_blocks; i++) {
      if (inv[i] == -1) {
         inv[i] = j++;
         vector_append(block_map, i);
      }
   }

   for (i = 0; i < num_blocks; i++)
      if (i != inv[i]) {
         off64_t holding = inv[i], vacant = i;
         fseeko(file, i * block_size, SEEK_SET);
         fread(tmp1, block_size, 1, file);
         while (holding != vacant) {
            j = vector_get(block_map, vacant);
            fseeko(file, j * block_size, SEEK_SET);
            bp_read(file, tmp2, block_size, 1);
            fseeko(file, vacant * block_size, SEEK_SET);
            bp_write(file, tmp2, block_size, 1);
            inv[vacant] = vacant;
            vector_set(block_map, vacant, vacant);
            vacant = j;
         }
         fseeko(file, vacant * block_size, SEEK_SET);
         bp_write(file, tmp1, block_size, 1);
         vector_set(block_map, holding, vacant);
         inv[vacant] = holding;
      }

   free(tmp1);
   free(tmp2);
   free(inv);
   vector_destroy(block_map);
   free(block_map);
}

int main(int argc, char *argv[])
{
   FILE *infile;
   struct indices indices;
   struct poliqarp_cdf_info cdf;
   struct vector *block_map;
   size_t i, block_size;
   off64_t index_size;

   bp_init("bpindexer");
   cmdline_process(argc, argv);
   open_files(&infile);
   if (poliqarp_cdf_read(cmdline.input_file, &cdf) != 0) {
      if (errno == ENOENT) {
         bp_warning("CDF file is missing; either this is a version 1 corpus or an incomplete corpus");
         bp_fatal("indexing corpora of version %d is not supported", cdf.version);
      } else
         bp_system_error("could not read CDF file");
   }
   if (cdf.version != POLIQARP_ABI_VERSION)
      bp_fatal("indexing corpora of version %d is not supported", cdf.version);
   if (cdf.endianness == ENDIAN_BE)
      bp_fatal("indexing big-endian corpora is not supported");
   cdf.indices = cmdline.indices;
   cdf.granularity = cmdline.granularity;
   indices = process_segments(infile);
   for (i = 0; i < indices.size; i++) {
      struct occurrences *occurrences = indices.indices[i].occurrences;
      FILE *outfile = indices.indices[i].outfile;
      bp_printf("Creating %s...\n", descriptions[indices.indices[i].type].
         description);
      block_size = occurrences->bs_backend.block_size;
      index_size = merge_and_recode(outfile, indices.indices + i, &block_map);
      permute(outfile, block_size, block_map);
      bp_truncate(outfile, index_size);
      bp_close(outfile);
      bp_printf("Index created, size = %" PRIdMAX " bytes.\n", (intmax_t)index_size);
   }
   if (poliqarp_cdf_write(cmdline.input_file, &cdf) != 0)
      bp_system_error("could not write CDF file");
   return 0;
}

