/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

#include <foostring/foostring.h>
#include <unibits/strcoll.h>

#include "binary-pool.h"

#include <sakura/common/newdict.h>

int binary_pool_compare_freq(const void *p1, const void *p2)
{
   const struct binary_pool_node *n1 = p1;
   const struct binary_pool_node *n2 = p2;

   return n1->count == n2->count ? n1->length - n2->length : 
      n2->count - n1->count;
}

int binary_pool_compare_alpha(const void *p1, const void *p2)
{
   const struct binary_pool_node *n1 = p1;
   const struct binary_pool_node *n2 = p2;
   
   return unibits_afronte_strcoll(n1->data, n2->data);
}

int binary_pool_compare_ialpha(const void *p1, const void *p2)
{
   const struct binary_pool_node *n1 = p1;
   const struct binary_pool_node *n2 = p2;

   return unibits_afronte_stricoll(n1->data, n2->data);
}

int binary_pool_compare_atergo(const void *p1, const void *p2)
{
   const struct binary_pool_node *n1 = p1;
   const struct binary_pool_node *n2 = p2;

   return unibits_atergo_strcoll(n1->data, n2->data);
}

int binary_pool_compare_iatergo(const void *p1, const void *p2)
{
   const struct binary_pool_node *n1 = p1;
   const struct binary_pool_node *n2 = p2;

   return unibits_atergo_stricoll(n1->data, n2->data);
}

/* hash given key */
static inline size_t binary_hash(const unsigned char *key, size_t length)
{
   size_t hash; 
   const unsigned char *end = key + length;
   for (hash = 0; key < end; ++key)
      hash = 37 * hash + *key;
   return hash;
}

/* initialize pool structs */
void binary_pool_init(struct binary_pool *this, struct marena *arena, 
   size_t size)
{
   size_t i;

   this->hash_array = malloc(sizeof *this->hash_array * size);
   this->array_size = size;
   this->num_items = 0;
   this->dense_array = NULL;
   this->index = NULL;
   this->offset = NULL;
   this->last = NULL;
   this->arena = arena;
   for (i = 0; i < this->array_size; ++i)
      this->hash_array[i].node.count = 0;
}

void binary_pool_free(struct binary_pool *this)
{
   free(this->hash_array);
   free(this->dense_array);
   free(this->index);
   free(this->offset);
}

int binary_pool_add_dict(struct binary_pool *pool, const char *image_pathname,
   const char *offset_pathname)
{
   struct newdict dict;
   int status;
   size_t i;
   size_t num_items;

   status = newdict_open(&dict, image_pathname, offset_pathname, false);
   if (status != 0)
      return status;
   num_items = GET_NUM_ITEMS(&dict);
   for (i = 0; i < num_items; ++i)
      binary_pool_add(pool, GET_ITEM(&dict, i), GET_LENGTH(&dict, i));
   newdict_close(&dict);
   return 0;
}

int binary_pool_add_dict_freq(struct binary_pool *pool, 
   const char *image_pathname, const char *offset_pathname, 
   const char *freq_pathname)
{
   struct newdict dict;
   int status;
   size_t i;
   size_t num_items;

   status = newdict_open2(&dict, image_pathname, offset_pathname, 
      freq_pathname);
   if (status != 0)
      return status;
   num_items = GET_NUM_ITEMS(&dict);
   for (i = 0; i < num_items; ++i)
      binary_pool_add_freq(pool, GET_ITEM(&dict, i), GET_LENGTH(&dict, i), 
         GET_FREQ(&dict, i));
   newdict_close(&dict);
   return 0;
}

struct binary_pool_node *binary_pool_add(struct binary_pool *this,
   const void *buffer, size_t length)
{
   return binary_pool_add_freq(this, buffer, length, 1);
}

struct binary_pool_node *binary_pool_add_freq(struct binary_pool *this,
   const void *buffer, size_t length, size_t freq)
{
   size_t hash;
   struct binary_pool_chain *chain;
   struct binary_pool_chain *last;

   /* calculate hash */
   hash = binary_hash(buffer, length);

   /* check last accessed node */
   if ((chain = this->last) != NULL && chain->node.length == length &&
      memcmp(chain->node.data, buffer, length) == 0) 
   {
      chain->node.count++;
      return &chain->node;
   }

   /* get initial node */
   chain = this->hash_array + hash % this->array_size;

   /* empty initial node check */
   if (chain->node.count == 0) {
      chain->next = NULL;
      this->last = chain;
      goto fill_values;
   }

   /* try to match the keys */
   for (last = chain; chain != NULL; chain = chain->next) {
      int cmp_result;

      /* non equal hash or data length -> skip */
      if (chain->node.length != length)
         continue;

      /* compare binaries here */
      cmp_result = memcmp(chain->node.data, buffer, length);

#ifdef USE_ORDERED_LIST
      /* key ordering broken -> break and create new node */
      if (cmp_result > 0)
         break;
#endif
      /* identical keys -> increase count and return */
      if (cmp_result == 0) {
         chain->node.count += freq;
         return &chain->node;
      }
      last = chain;
   }
   /* create new node (between 'last' and 'node')  */
   this->last = chain = marena_alloc(this->arena, sizeof *chain);
   chain->next = last->next;
   last->next = chain;

fill_values:

   /* fill the values of node */
   chain->node.id = this->num_items++;
   chain->node.count = freq;
   chain->node.length = length;
   chain->node.data = marena_alloc(this->arena, length);
   memcpy(chain->node.data, buffer, length);
   return &chain->node;
}

/* add object */
struct binary_pool_node *binary_pool_add_string(struct binary_pool *this,
   const void *buffer, size_t length)
{
   char *s;
   struct binary_pool_node *res;

   s = malloc(length + 1); /* DJ: changed alloca to malloc; unsafe? */
   memcpy(s, buffer, length);
   s[length] = 0;
   res = binary_pool_add(this, s, length + 1);
   free(s);
   return res;
}

struct binary_pool_node *binary_pool_add_string2(struct binary_pool *this,
   const char *str)
{
   return binary_pool_add(this, (const void *)str, strlen(str) + 1);
}

struct binary_pool_node *binary_pool_add_string_lowercase(
   struct binary_pool *this, const char *str)
{
   char *copy = strdup(str);
   char *tmp = copy;
   struct binary_pool_node *result;
   while (*tmp) {
      *tmp = ascii_tolower(*tmp);
      tmp++;
   }
   result = binary_pool_add_string2(this, copy);
   free(copy);
   return result;
}

struct binary_pool_node *binary_pool_lookup(struct binary_pool *this,
   const void *buffer, size_t length)
{
   size_t hash;
   struct binary_pool_chain *chain;

   /* calculate hash */
   hash = binary_hash(buffer, length);
   /* check last accessed node */
   if ((chain = this->last) != NULL && chain->node.length == length &&
      memcmp(chain->node.data, buffer, length) == 0) {
      return &chain->node;
   }
   /* get initial node */
   chain = this->hash_array + hash % this->array_size;
   /* empty initial node check */
   if (chain->node.count == 0)
      return NULL;
   /* try to find match the keys */
   for (; chain != NULL; chain = chain->next) {
      int cmp_result;
      /* non-equal hash or data length -> skip */
      if (chain->node.length != length)
         continue;
      /* compare binary here */
      cmp_result = memcmp(chain->node.data, buffer, length);
#ifdef USE_ORDERED_LIST
      /* key ordering broken -> break and create new node */
      if (cmp_result > 0)
         return NULL;
#endif
      /* identical keys -> increase count and return */
      if (cmp_result == 0) {
         return &chain->node;
      }
   }
   return NULL;
}

/* lookup string */
struct binary_pool_node *binary_pool_lookup_string(struct binary_pool *this,
   const void *buffer, size_t length)
{
   char *s;
   struct binary_pool_node *res;

   s = malloc(length + 1); /* DJ: changed alloca to malloc; unsafe? */
   memcpy(s, buffer, length);
   s[length] = 0;
   res = binary_pool_lookup(this, s, length + 1);
   free(s);
   return res;
}

/* create dense array of nodes good for sorting */
void binary_pool_make_dense_array(struct binary_pool *this)
{
   size_t i;
   struct binary_pool_chain *chain;
   struct binary_pool_chain *hash_array;
   struct binary_pool_node *dense_array;

   assert(this->dense_array == NULL);

   /* allocate dense array */
   this->dense_array = malloc(sizeof *this->dense_array * this->num_items);

   /* shortcuts */
   hash_array = this->hash_array;
   dense_array = this->dense_array;

   /* iterate over entire hash table and copy all elements */
   for (i = 0; i < this->array_size; ++i)
      for (chain = hash_array + i; chain && chain->node.count > 0; 
         chain = chain->next)
      {
         dense_array[chain->node.id] = chain->node;
      }

   /* allocate index and offset */
   this->index = malloc(sizeof *this->index * this->num_items);
   this->offset = malloc(sizeof *this->offset * this->num_items);

   /* reset index to default values */
   binary_pool_reset_index(this);
}

/* reset index */
void binary_pool_reset_index(struct binary_pool *this)
{
   size_t i;

   for (i = 0; i < this->num_items; ++i)
      this->index[i] = i;
}

/* sort data according to given comparator */
void binary_pool_sort(struct binary_pool *this, compar_fn_t comparator)
{
   qsort(this->dense_array, this->num_items, sizeof *this->dense_array,
      comparator);
}

/* sort index according to given comparator */
void binary_pool_sort_index(struct binary_pool *this, compar_fn_t comparator)
{
   size_t i;
   qsort(this->dense_array, this->num_items, sizeof *this->dense_array, 
      comparator);
   for (i = 0; i < this->num_items; ++i)
      this->index[this->dense_array[i].id] = i;
}

/* write index to stream */
void binary_pool_write_index(struct binary_pool *this, FILE * stream)
{
   fwrite(this->index, sizeof *this->index, this->num_items, stream);
}

/* write data to stream, build offsets */
void binary_pool_write_data(struct binary_pool *this, FILE * stream)
{
   struct binary_pool_node *item;
   size_t i;
   size_t offset;
   
   for (offset = 0, i = 0; i < this->num_items; ++i) {
      item = this->dense_array + i;
      offset += sizeof(item->length); /* remember about length */
      this->offset[item->id] = offset;
      fwrite(&item->length, sizeof item->length, 1, stream);
      fwrite(item->data, 1, item->length, stream);
      /* align to boundary */
      switch (item->length % BINARY_POOL_ALIGNMENT) {
         case 1:
            fputc(0, stream); ++offset;
         case 2:
            fputc(0, stream); ++offset;
         case 3:
            fputc(0, stream); ++offset;
         default:
            break;
      }
      offset += item->length;
   }
}

/* write offsets to stream */
void binary_pool_write_offsets(struct binary_pool *this, FILE *stream)
{
   fwrite(this->offset, sizeof *this->offset, this->num_items, stream);
}

void binary_pool_debug(struct binary_pool *this, struct viewport *vp)
{
   size_t i;
   size_t len;
   struct binary_pool_chain *chain;
   size_t longest_chain = 0;
   size_t num_chained_items = 0;
   size_t num_chains = 0;
   size_t num_empty_buckets = 0;

   for (i = 0; i < this->array_size; ++i) {
      for (len = 0, chain = this->hash_array + i; 
         chain && chain->node.count > 0; chain = chain->next)
      {
         ++len;
      }
      if (len == 0)
         ++num_empty_buckets;
      else if (len > 1) {
         if (len > longest_chain)
            longest_chain = len;
         num_chained_items += len;
         ++num_chains;
      }
   }
   vp_nest(vp);
   if (this->num_items > 0) {
      vp_printf(_("Number of buckets:       %d\n"), this->array_size);
      vp_printf(_("Number of empty buckets: %d (%.2f%% of all buckets)\n"), 
         num_empty_buckets, 100.0 * num_empty_buckets / this->array_size);
      vp_printf(_("Number of unique items:  %d\n"), this->num_items);
      vp_printf(_("Number of chains:        %d "
         "(%.2f%% of all buckets were chained)\n"), num_chains, 
         100.0 * num_chains / this->array_size);
      vp_printf(_("Number of chained items: %d "
         "(%.2f%% of all items were chained)\n"), num_chained_items, 
         100.0 * num_chained_items / this->num_items);
      vp_printf(_("Length of longest chain: %d\n"), longest_chain);
      if (num_chains > 0)
      vp_printf(_("Average chain length:    %.2f\n"), 
         (double)num_chained_items / num_chains);
      vp_printf(_("Longest item:            %ld\n"), 
         binary_pool_longest_item(this));
      {
         size_t foo[] = { 2, 3, 4, 5, 100 };
         size_t i;
         for (i = 0; i < sizeof foo / sizeof *foo; ++i)
            vp_printf(_("Number of items less frequent than %d:    %ld\n"), 
               foo[i], binary_pool_num_less_freq_than(this, foo[i]));
      }
      {
         size_t foo[] = { 1, 3, 4, 5, 100 };
         size_t i;
         for (i=0; i<sizeof foo / sizeof *foo; ++i)
            vp_printf(_("Number of items more frequent than %d:    %ld\n"), 
               foo[i], binary_pool_num_more_freq_than(this, foo[i]));
      }
   } else
      vp_printf(_("The array is empty\n"));
   vp_unnest(vp);
}

size_t binary_pool_longest_item(struct binary_pool *this)
{
   size_t i;
   size_t best = 0;
   assert(this->dense_array != NULL);
   for (i = 0; i < this->num_items; ++i)
      if (this->dense_array[i].length > best)
         best = this->dense_array[i].length;
   return best;
}

size_t binary_pool_num_less_freq_than(struct binary_pool *this, size_t freq)
{
   size_t i;
   size_t num = 0;
   assert(this->dense_array != NULL);
   for (i = 0; i < this->num_items; ++i)
      if (this->dense_array[i].count < freq)
         ++num;
   return num;
}

size_t binary_pool_num_more_freq_than(struct binary_pool *this, size_t freq)
{
   size_t i;
   size_t num = 0;
   assert(this->dense_array != NULL);
   for (i = 0; i < this->num_items; ++i)
      if (this->dense_array[i].count > freq)
         ++num;
   return num;
}

const void *binary_pool_get_by_id(const struct binary_pool *this, 
   size_t id)
{
   size_t i;
   struct binary_pool_chain *chain;

   /* FIXME: use dense array if it is available but not sorted yet */

   for (i = 0; i < this->array_size; ++i)
      for (chain = this->hash_array + i; chain && chain->node.count > 0; 
         chain = chain->next)
      {
         if (id == chain->node.id)
            return chain->node.data;
      }
   
   return NULL;
}
