/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <assert.h>
#include <errno.h>
#include <locale.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

#if HAVE_OPENMP
#include <omp.h>
#endif

#include <foostring/foostring.h>

#include <sakura/common/getline.h>
#include <sakura/poliqarp.h>
#include <sakura/exception.h>

#include <bp/io.h>
#include <bp/ng/command-line.h>
#include <bp/ng/configuration.h>
#include <bp/ng/document.h>
#include <bp/ng/fs-walk.h>
#include <bp/ng/list.h>
#include <bp/ng/xml-parser.h>

#include <bp/common.h>

static struct list documents;
static struct configuration configuration;
static struct corpus corpus;

static int find_xml_name(const char *file_name, const char *dir_name)
{
   struct stat statbuf;
   char *path = string_aformat("%s/%s", dir_name, file_name);
   bool found = stat(path, &statbuf) == 0 && S_ISREG(statbuf.st_mode);
   free(path);
   return !found;
}

static int visit_dir(const char *path, void *extra)
{
   size_t i;
   const char *morph_name = NULL, *header_name = NULL;
   for (i = 0; i < configuration.morph_names.length; i++) {
      morph_name = configuration.morph_names.items[i];
      if (find_xml_name(morph_name, path) == 0)
         break;
      else
         morph_name = NULL;
   }
   for (i = 0; i < configuration.header_names.length; i++) {
      header_name = configuration.header_names.items[i];
      if (find_xml_name(configuration.header_names.items[i], path) == 0)
         break;
      else
         header_name = NULL;
   }
   if ((header_name == NULL) && (morph_name == NULL))
      return 0;
   if ((header_name == NULL) || (morph_name == NULL)) {
      if (morph_name != NULL)
         bp_warning("%s/%s found but no corresponding header file", path, morph_name);
      else
         bp_warning("%s/%s found but no corresponding morphosyntax file", path, header_name);
   }
   else {
      struct document *document = document_alloc(path, morph_name, header_name);
      list_append(&documents, document);
   }
   return 0;
}

void presetup_locale()
{
   setlocale(LC_ALL, "");
}

void setup_locale()
{
   const char *locale;
   struct poliqarp_error error = poliqarp_error_none;
   locale = setlocale(LC_COLLATE, configuration.locale);
   if (locale == NULL) {
      /* Maybe UTF-8 variant would be fine? */
      char *new_locale = bp_malloc(strlen(configuration.locale) + 7);
      char *old_locale_end = configuration.locale;
      char *new_locale_end = new_locale;
      while (*old_locale_end != '\0' && *old_locale_end != '.')
         *new_locale_end++ = *old_locale_end++;
      strcpy(new_locale_end, ".UTF-8");
      if (setlocale(LC_COLLATE, new_locale) == NULL)
         bp_fatal("Unable to set a locale");
   }
   if (poliqarp_create(NULL /* use the current locale */, &error) != 0)
      bp_fatal(poliqarp_error_message_get(&error));
   locale = setlocale(LC_COLLATE, NULL);
   if (locale == NULL)
      bp_fatal("unable to check current string collation locale");
   bp_printf("Current string collation locale is %s.\n", locale);
   if (strcmp(locale, "C") == 0 || strcmp(locale, "POSIX") == 0) {
      bp_warning("current string collation locale is ASCII-oriented; "
         "this is probably NOT what you want", locale);
   }
}

#define BP_DOCS_ACCEPTED_FORMAT "%s.bp.docs.accepted"
#define BP_DOCS_REJECTED_FORMAT "%s.bp.docs.rejected"

int main(int argc, char *argv[])
{
   FILE *fp_accepted, *fp_rejected;
   size_t i;
   string_t path = string_create();

   documents = list_empty;

   presetup_locale();

   bp_init("bpng");
   xml_init();

   command_line_process(argc, argv);
#if HAVE_OPENMP
   if (!command_line_data.parallel)
      omp_set_num_threads(1);
#endif
   configuration_create(&configuration, command_line_data.base_name);

   if (command_line_data.incremental) {
      FILE *fp;
      char *buffer = NULL;
      size_t buffer_length = 0;
      bp_printf("Adding previously rejected documents...\n");
      string_format(path, BP_DOCS_REJECTED_FORMAT, command_line_data.base_name);
      fp = bp_open(path, "rt");
      while (1) {
         ssize_t line_length = getline(&buffer, &buffer_length, fp);
         if (line_length <= 0)
            break;
         if (buffer[line_length - 1] == '\n')
            buffer[line_length - 1] = '\0';
         visit_dir(buffer, NULL);
      }
      if (!feof(fp))
         bp_system_error("read error");
      bp_close(fp);
   }

   setup_locale();

   bp_printf("Scanning...\n");
   for (i = 0; i < command_line_data.root_dirs.length; i++) {
      const char *path = command_line_data.root_dirs.items[i];
      fs_walk(path, visit_dir, NULL);
   }

   bp_printf("Number of documents to process: %" PRIuSIZE "\n", documents.length);

   string_format(path, BP_DOCS_ACCEPTED_FORMAT, command_line_data.base_name);
   fp_accepted = bp_open(path, command_line_data.incremental ? "at" : "wt");
   string_format(path, BP_DOCS_REJECTED_FORMAT, command_line_data.base_name);
   fp_rejected = bp_open(path, "wt");
   string_free(path);

   corpus_create(&corpus, command_line_data.base_name,
      command_line_data.incremental);
   configuration_commit(&configuration, &corpus);
   #pragma omp parallel for private(i) schedule(dynamic)
   for (i = 0; i < documents.length; i++) {
      int rc;
      struct document *document = documents.items[i];
      bp_printf("+ %s\n", document->dir_name);
      rc = 
         document_parse_header(document, &configuration.meta_keys) ||
         document_parse_morph(document, &corpus);
      document_free_header(document);
      /* If a physical document (i.e. an XML-file) contains multiple logical
       * documents, the following code is totally unreliable: some logical
       * subdocuments may have been already committed to the binary corpus. */
      #pragma omp critical
      {
         if (rc == 0)
            fprintf(fp_accepted, "%s\n", document->dir_name);
         else
            fprintf(fp_rejected, "%s\n", document->dir_name);
      }
   }
   corpus_commit(&corpus);
   corpus_destroy(&corpus);
   bp_close(fp_accepted);
   bp_close(fp_rejected);

   bp_printf("Done.\n");

   xml_cleanup();
   list_destroy(&documents, document_free);
   configuration_destroy(&configuration);
   command_line_destroy();

   return 0;
}
