/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <poliqarp-config.h>

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>

#include <assert.h>
#include <errno.h>
#include <time.h>

#include <dirent.h>
#include <locale.h>

#include <sakura/poliqarp.h>
#include <sakura/cdf.h>

#include <bp/common.h>

#include "cmdline.h"
#include "utils.h"
#include "foolog.h"
#include "list.h"
#include "viewport.h"

#include "morph-parser.h"
#include "header-parser.h"
#include "syntax-parser.h"
#include "chunk-merger.h"

list_t good_morph_list = NULL;
list_t good_header_list = NULL;
list_t good_dir_list = NULL;

static string_t probed_file = NULL;

static int probe_file(const string_t file, const char *ext, const char *dirname)
{
   int result;
   string_t s = string_init(dirname);
   string_append_str(s, "/");
   string_append(s, file);
   if (ext)
      string_append_str(s, ext);
   result = (access(string_str(s), R_OK | F_OK) == 0);
   if (result)
      probed_file = s;
   else
      string_free(s);
   return result;
}

static int find_filtered_file(const string_t elem, const char *dirname)
{
   return 
      (probe_file(elem, NULL, dirname)  ||
       probe_file(elem, ".gz", dirname) ||
       probe_file(elem, ".bz2", dirname));
}

static int dirname_try_predicate(const void *elem_, const void *dirname_)
{
   const string_t elem = (const string_t)elem_;
   const char *dirname = (const char *)dirname_;

   return !find_filtered_file(elem, dirname);
}

void dirname_try(const char *dirname)
{
   string_t f_morph = NULL, f_header = NULL;
   char *f_ignore = string_aformat("%s/.ignore", dirname);
   bool have_ignore = (access(f_ignore, F_OK) == 0);
   free(f_ignore);
   if (have_ignore) {
      foolog(MSG_INFO, "Ignoring directory %s", dirname);
      return;
   }
   if (list_find2(cmdline.morph_names, dirname_try_predicate, dirname))
      f_morph = probed_file;
   if (list_find2(cmdline.header_names, dirname_try_predicate, dirname))
      f_header = probed_file;
   if (f_header && f_morph) {
      good_dir_list = list_prepend(good_dir_list, 
         list_create(string_init(dirname)));
      good_morph_list = list_prepend(good_morph_list, list_create(f_morph));
      good_header_list = list_prepend(good_header_list, list_create(f_header));
   } else if (f_header) {
      foolog(MSG_WARNING, "Only header was found in %s", dirname);
      string_free(f_header);
   } else if (f_morph) {
      foolog(MSG_WARNING, "Only morph was found in %s", dirname);
      string_free(f_morph);
   }
}

void walk_dir(const char *dirname)
{
   DIR *dir = opendir(dirname);
   struct dirent *entry;
   if (dir == NULL) {
      foolog(MSG_WARNING, "Unable to descend to directory %s", dirname);
      return;
   }
   dirname_try(dirname);
   while ((entry = readdir(dir)) != NULL) {
      char *fullname;
      struct stat statbuf;
      if (strcmp(entry->d_name, ".") == 0 ||
          strcmp(entry->d_name, "..") == 0) 
      {
         continue;
      }
      fullname = string_aformat("%s/%s", dirname, entry->d_name);
      if (stat(fullname, &statbuf) == 0 && S_ISDIR(statbuf.st_mode))
         walk_dir(fullname);
      free(fullname);
   }
   closedir(dir);
}

static char* getcwda()
{
   char *buffer, *path;
   size_t size = PATH_MAX;
   buffer = malloc(size);
   while (1)
   {
      if (buffer == NULL)
         foolog(MSG_FATAL, "Out of memory");
      path = getcwd(buffer, size);
      if (path != NULL)
         break;
      if (errno != ERANGE)
         foolog(MSG_FATAL, "Unable to get current working directory name");
      size *= 2;
      buffer = realloc(buffer, size);
   }
   return path;
} 

int main(int argc, char *argv[])
{
   struct poliqarp_cdf_info cdf;
   struct poliqarp_error error = poliqarp_error_none;

   bp_parser *morph_parser;
   bp_parser *header_parser;
   bp_parser *chunk_parser;
   bp_parser *syntax_parser;
   bp_parser_env parser_env;
   list_t n;

   /* initialize viewports */
   vp_glob_init();
   vp_prefix(vp_out, VP_NORMAL);
   vp_prefix(vp_err, VP_ERROR);

   bp_init("bp");
   if (poliqarp_create("" /*use the system locale*/, &error) != 0) {
      foolog(MSG_FATAL, poliqarp_error_message_get(&error));
   }


   /* parse input */
   cmdline_process(argc, argv);

   /* let user have a chance to verify his LC_COLLATE settings */
   {
      char *locale = setlocale(LC_COLLATE, NULL);
      if (locale == NULL)
         foolog(MSG_FATAL, "Unable to check current string collation locale");
      else if (strcmp(locale, "C") == 0 || strcmp(locale, "POSIX") == 0)
         foolog(MSG_WARNING, "Current string collation locale is %s. "
            "This is probably NOT what you want.", locale);
      else
         foolog(MSG_INFO, "Current string collation locale is %s.", locale);
   }

  
   if (string_ccmp(cmdline.dirname, ".") != 0) {
      /* don't even try to create corpora outside the current working directory */
      foolog(MSG_FATAL, "Creating corpora outside the current working directory is not supported.");
   }

   /* don't even try to create big-endian corpora */
#ifdef WORDS_BIGENDIAN
   foolog(MSG_FATAL, "big-endian machines are (currently) not supported\n");
   /* FIXME */
#endif

   /* create env */
   create_bp_parser_env(&parser_env);

   /* create parsers */
   foolog(MSG_INFO, "Initializing syntax parser");
   vp_nest(vp_out);
   syntax_parser = syntax_parser_create(string_str(cmdline.basename));
   vp_unnest(vp_out);

   foolog(MSG_INFO, "Initializing morph parser");
   vp_nest(vp_out);
   morph_parser = morph_parser_create(syntax_parser, 
      string_str(cmdline.basename), cmdline.incremental);
   vp_unnest(vp_out);

   foolog(MSG_INFO, "Initializing header parser");
   vp_nest(vp_out);
   header_parser = header_parser_create(string_str(cmdline.basename), cmdline.incremental);
   vp_unnest(vp_out);

   foolog(MSG_INFO, "Initializing chunk parser");
   vp_nest(vp_out);
   chunk_parser = chunk_parser_create(string_str(cmdline.basename), cmdline.incremental);
   vp_unnest(vp_out);

   /* preprocessing stage */
   if (cmdline.incremental) {
      foolog(MSG_INFO, "Incremental building requires preprocessing stage");
      vp_nest(vp_out);
      morph_parser->preprocess(morph_parser);
      header_parser->preprocess(header_parser);
      vp_unnest(vp_out);
   } else
      foolog(MSG_INFO, "Not doing incremental build this time, skipping preprocessing stage");

   /* scanning stage */

   /* scanning stage */
   foolog(MSG_INFO, "Scanning stage");
   vp_nest(vp_out);
   for (n = cmdline.root_dirs; n != NULL; n = n->next)
      walk_dir(string_str((string_t)(n->item)));
   vp_unnest(vp_out);
   /* After scanning, invert good dir/morph/header lists so that they
    * will be visited in finding order. */
   good_morph_list = list_revert(good_morph_list);
   good_header_list = list_revert(good_header_list);
   good_dir_list = list_revert(good_dir_list);

   /* parsing stage */
   foolog(MSG_INFO, "Processing morph and syntax files...");
   vp_nest(vp_out);
   for (n = good_morph_list; n != NULL; n = n->next) {
      char *current_dir;
      bool syntax = false;
      string_t target_dir, target_file, syntax_file = string_init("syntax.xml");
      split_pathname((string_t)n->item, &target_dir, &target_file);
      current_dir = getcwda();
      vp_nest(vp_out);
      chdir(string_str(target_dir));
      if (find_filtered_file(syntax_file, ".")) {
         string_t adir, afile;
         split_pathname(probed_file, &adir, &afile);
         syntax = true;
         syntax_parser->work(&parser_env, syntax_parser, string_str(probed_file), 
            string_str(target_dir), string_str(afile));
         free(probed_file);
         free(adir);
         free(afile);
      }
      morph_parser->work(&parser_env, morph_parser, string_str((string_t)n->item), 
         string_str(target_dir), string_str(target_file));
      if (syntax)
         syntax_parser->postprocess(syntax_parser);
      chdir(current_dir);
      vp_unnest(vp_out);
      string_free(target_dir);
      string_free(target_file);
      string_free(syntax_file);
      free(current_dir);
   }
   vp_unnest(vp_out);
   foolog(MSG_INFO, "Processing header files...");
   vp_nest(vp_out);
   for (n = good_header_list; n != NULL; n = n->next) {
      char *current_dir;
      string_t target_dir, target_file;
      split_pathname((string_t)n->item, &target_dir, &target_file);
      current_dir = getcwda();
      vp_nest(vp_out);
      chdir(string_str(target_dir));
      header_parser->work(&parser_env, header_parser, string_str((string_t)n->item), 
         string_str(target_dir), string_str(target_file));
      chdir(current_dir);
      vp_unnest(vp_out);
      string_free(target_dir);
      string_free(target_file);
      free(current_dir);
   }
   vp_unnest(vp_out);

   /* postprocessing stage */
   vp_printf("Postprocessing stage\n");
   vp_nest(vp_out);
   morph_parser->postprocess(morph_parser);
   header_parser->postprocess(header_parser);
   vp_unnest(vp_out);

   if (cmdline.show_parser_stats) {
      morph_parser->stats(morph_parser);
      header_parser->stats(header_parser);
   }

   /* merging stage */
   vp_printf("Merging corpus...\n");
   vp_nest(vp_out);
   for (n = good_morph_list; n != NULL; n = n->next) {
      char *current_dir;
      string_t target_dir, target_file;
      split_pathname((string_t)n->item, &target_dir, &target_file);
      current_dir = getcwda();
      chdir(string_str(target_dir));
      morph_parser->merge(&parser_env, morph_parser, string_str((string_t)n->item), 
         string_str(target_dir), string_str(target_file));
      chdir(current_dir);
      string_free(target_dir);
      string_free(target_file);
      free(current_dir);
   }
   vp_unnest(vp_out);

   vp_printf("Merging meta-data...\n");
   vp_nest(vp_out);
   for (n = good_header_list; n != NULL; n = n->next) {
      char *current_dir;
      string_t target_dir, target_file;
      split_pathname((string_t)n->item, &target_dir, &target_file);
      current_dir = getcwda();
      chdir(string_str(target_dir));
      header_parser->merge(&parser_env, header_parser, string_str((string_t)n->item), 
         string_str(target_dir), string_str(target_file));
      chdir(current_dir);
      string_free(target_dir);
      string_free(target_file);
      free(current_dir);
   }
   vp_unnest(vp_out);

   vp_printf("Creating chunks...\n");
   vp_nest(vp_out);
   for (n = good_header_list; n != NULL; n=n->next) {
      char *current_dir;
      string_t target_dir, target_file;
      split_pathname((string_t)n->item, &target_dir, &target_file);
      current_dir = getcwda();
      chdir(string_str(target_dir));
      chunk_parser->merge(&parser_env, chunk_parser, string_str((string_t)n->item), 
         string_str(target_dir), string_str(target_file));
      chdir(current_dir);
      string_free(target_dir);
      string_free(target_file);
      free(current_dir);
   }
   vp_unnest(vp_out);

   /* gather errors */
   parser_env.num_errors += morph_parser->num_errors;
   parser_env.num_errors += header_parser->num_errors;
   parser_env.num_errors += chunk_parser->num_errors;
   parser_env.num_errors += syntax_parser->num_errors;
   foolog(MSG_INFO, "Corpus construction finished");
   if (parser_env.num_errors == 0)
      foolog(MSG_INFO, "There were no errors");
   else
      foolog(MSG_INFO, ngettext("There was %d error", "There were %d errors", 
         parser_env.num_errors), parser_env.num_errors);

   /* destroy parsers */
   morph_parser->destroy(morph_parser);
   header_parser->destroy(header_parser);
   syntax_parser->destroy(syntax_parser);

   /* destroy env */
   destroy_bp_parser_env(&parser_env);

   /* create cdf file */
   poliqarp_cdf_create(&cdf);
   if (poliqarp_cdf_write(string_str(cmdline.basename), &cdf) == -1)
      foolog(MSG_FATAL, "Could not write CDF file");

   /* destroy viewports */
   vp_glob_free();
   return EXIT_SUCCESS;
}

