/*
 * This file is part of the Poliqarp suite.
 * 
 * Copyright (C) 2004-2009 by Instytut Podstaw Informatyki Polskiej
 * Akademii Nauk (IPI PAN; Institute of Computer Science, Polish
 * Academy of Sciences; cf. www.ipipan.waw.pl).  All rights reserved.
 * 
 * This file may be distributed and/or modified under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation and appearing in the file gpl.txt included in the packaging
 * of this file.  (See http://www.gnu.org/licenses/translations.html for
 * unofficial translations.)
 * 
 * A commercial license is available from IPI PAN (contact
 * Michal.Ciesiolka@ipipan.waw.pl or ipi@ipipan.waw.pl for more
 * information).  Licensees holding a valid commercial license from IPI
 * PAN may use this file in accordance with that license.
 * 
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING
 * THE WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE.
 */

/**
 * @file occurrences.h
 * @brief Set of occurrences.
 *
 * The first phase of indexing is performed by artificially splitting the
 * corpus into documents (or artificial documents) of equal size, and then 
 * gathering information about which terms occur in which documents. (A term
 * can be an orthographic form, a base form or a tag, depending on which index
 * we're building.) Each occurrence is stored only once, thanks to the BST
 * tree.
 * 
 * When the number of occurrences held in memory exceeds the limit, they are
 * sorted (with respect to number of document, and to number of term if the 
 * documents are the same) and stored compressed on disk. This amount of 
 * occurrences that fit in memory at a time is called a run. Therefore,
 * the first stage of indexing is about creating runs.
 */

#ifndef POLIQARP_OCCURRENCES_H
#define POLIQARP_OCCURRENCES_H

#include <stdio.h>
#include <sakura/common/bitstream.h>
#include <sakura/common/bs-file-backend.h>

#include "bst.h"
#include "vector.h"

/**
 * Occurrence of a term in a document.
 */
struct occurrence {
   uint32_t term;       /**< Number of term. */
   uint32_t document;   /**< Number of document that this term occurs in. */
};

/**
 * Array of occurrences, the main structure used during creation of runs.
 */
struct occurrences {
   struct occurrence *data;      /**< The array proper. */
   struct obs bs;                /**< The bitstream to which the occurrences
                                      are emitted when the structure gets
                                      flushed. */
   struct file_obs bs_backend;   /**< Backend of bs. */
   struct vector *frequencies;   /**< Vector storing number of occurrences for
                                      each term. */
   FILE *file;                   /**< File used by bs_backend. */
   size_t size;                  /**< Size of this array. */
   size_t used;                  /**< Number of occurrences currently used. */
   size_t remaining;             /**< When the processing is over, this field
                                      contains number of occurrences processed
                                      in the last run. */
   size_t num_documents;         /**< Total number of documents processed. */
   struct vector offsets;        /**< Vector storing offsets of the compressed
                                      runs in the file. */
};

/** 
 * Creates an empty array of occurrences.
 * @param occurrences the array to be created.
 * @param size size of the array.
 * @param file file to write runs to.
 */
void occurrences_create(struct occurrences *occurrences, size_t size,
   FILE *file);

/**
 * Frees all resources associated with an array of occurrences.
 * @param occurrences the array to be destroyed.
 */
void occurrences_destroy(struct occurrences *occurrences);

/**
 * Compares two occurrences. An occurrence is considered to be lesser than 
 * another one if it has lesser term number, or if it has lesser document number
 * in case the terms are the same.
 * @note This function can be used as a third argument to qsort().
 * @param x the first occurrence, cast to const void *.
 * @param y the second occurrence, cast to const void *.
 * @return a negative value if x < y, 0 if x == y, a positive value if x > y.
 */
int occurrences_compare(const void *x, const void *y);

/** 
 * Writes the occurrences currently stored in an array to disk, emptying the
 * array.
 * @param occurrences the array to be flushed.
 */
void occurrences_flush(struct occurrences *occurrences);

/**
 * Appends an occurrence to the array, flushing it if need be.
 * @param occurrences the array to insert new element to.
 * @param document number of document.
 * @param term number of term.
 */
void occurrences_insert(struct occurrences *occurrences, uint32_t document,
   uint32_t term);

/**
 * Causes all elements of a BST tree to be inserted in the array of occurrences,
 * considering them all to be number of terms that occur in a given document.
 * @param occurrences the array to dump elements to.
 * @param document number of document.
 * @param t root of the BST tree containing the terms.
 */
void occurrences_dump_segments(struct occurrences *occurrences, uint32_t document,
   struct bst_node *t);

#endif /* POLIQARP_OCCURRENCES_H */
