#!/usr/pkg/bin/python3.13

import sys
from os import system, path
import os
import shutil

from ..converter import *
from .grid import find_parameters
from .learner import *

VERSION = 0

__all__ = ['VERSION', 'train_converted_text', 'train_text', \
		'predict_text', 'predict_single_text', 'PredictionResult', 'TextModel', 'LIBLINEAR_HOME']


class PredictionResult(object):
	"""
	:class:`PredictionResult` is used to get predicted results generated by 
	:func:`predict_text` or :func:`predict_single_text`. Note that there are two types of 
	:class:`PredictionResult` instances. One can be used for subsequent analysis 
	(analyzable), and the other is not (`unanalyzable`). :func:`predict_text` returns an 
	`analyzable` result, while :func:`predict_single_text` returns an `unanalyzable` 
	result.
	"""
	
	_VERSION = 1
	
	def __init__(self, text_src = None, model_id = None, true_y = None, predicted_y = None, decvals = None, svm_file = None, labels = None, extra_svm_files = []):
		#: The location of the text file of the test data. The value is ``None`` for
		#: unanalyzable results.
		self.text_src = text_src
		#: The model ID. The value is ``None`` for unanalyzable results.
		self.model_id = model_id 
		#: The true labels. For analyzable results, it should be a :class:`list`,
		#: while for unanalyzable results, the value is ``None``.
		self.true_y = true_y
		#: The predicted labels. It should be a :class:`list`. However, it may also
		#: be a float value if it is returned by :func:`predict_single_text`.
		self.predicted_y = predicted_y
		#: Decision values of the results. For analyzable results, :attr:`decvals` is 
		#: from :func:`learner.predict`. Refer to :func:`learner.predict` for more 
		#: details. For unanalyzable results, :attr:`decvals` is a list of decision 
		#: values, sorted according to the :attr:`labels`.
		self.decvals = decvals
		#: Path of the LIBSVM-format test data. The value is ``None`` for
		#: unanalyzable results.
		self.svm_file = svm_file
		#: A list of labels. If the result is analyzable, it is a list of labels in the
		#: model.
		self.labels = labels

		#: A list of extra feature file paths. The list is empty for 
		#: unanalyzable results.
		self.extra_svm_files = extra_svm_files

	def get_accuracy(self):
		"""
		Return the accuracy of the prediction results. This method should be called 
		only if the results are analyzable.
		"""

		if not self.analyzable():
			raise RuntimeError('Only analyzable PredictionResult has accuracy.')

		true_y = self.true_y
		predicted_y = self.predicted_y
		l = len(self.true_y)
		return sum([true_y[i] == predicted_y[i] for i in range(l)]) / float(l)

	def save(self, file_name, analyzable = False, fmt = '.16g'):
		"""
		Save the model to a file.

		*analyzable* indicates whether the output result is analyzable. If the result
		is not analyzable, analyzable cannot be set to ``True``. If an analyzable 
		result is not saved in the analyzable mode, it will not be analyzable after 
		being reloaded.

		>>> print(p.analyzable())
		True
		>>> p.save('predict_result.not_analyzable', False)
		>>> p.load('predict_result.not_analyzable')
		>>> print(p.analyzable())
		False

		*fmt* is the output format for floating-point numbers. Fewer digits may result
		result in better readability and smaller result file size.
		"""

		fout = open(file_name, 'w')

		if type(self.predicted_y) is not list:
			raise ValueError('PredictionResult is only writable for single instance.')

		if analyzable is True and not self.analyzable():
			raise ValueError('PredictionResult is not analyzable if text_src or model_id is not given.')

		fout.write('version: {0}\n'.format(self._VERSION))
		fout.write('analyzable: {0}\n'.format(int(analyzable)))
		if analyzable:
			fout.write('text-src: {0}\n'.format(self.text_src))
			fout.write('extra-files:\t{0}\n'.format('\t'.join(self.extra_svm_files)))
			fout.write('model-id: {0}\n'.format(self.model_id))
		fout.write('\n')

		if not analyzable:
			for y in self.predicted_y:
				fout.write(y + "\n")
			fout.close()
			return

		fmt = '\t{{0:{0}}}'.format(fmt)
		for i in range(len(self.predicted_y)):
			fout.write("{py}\t{y}".format(py = self.predicted_y[i], y = self.true_y[i]))
			for v in self.decvals[i]:
				fout.write(fmt.format(v))
			fout.write('\n')

		fout.close()	
	
	def load(self, file_name):
		"""
		Load the result from a file.

		:class:`PredictionResult` cannot be initialized by specifying the result file 
		name. To create a :class:`PredictionResult` instance from a file, use

		>>> p = PredictionResult()
		>>> p.load('predict_result')
		"""

		self.text_src = None
		self.model_id = None
		self.true_y = None
		self.predicted_y = None
		self.decvals = None
		self.svm_file = None
		self.labels = None
		self.extra_svm_files = []

		fin = open(file_name, 'r')

		analyzable = False

		for line in fin:
			line = line.strip()
			if line == '':
				break
			
			if line.startswith('version: '):
				v = float(line.split(None, 1)[1])
				if self._VERSION < v:
					raise Exception("This result is not supported in this version. Please update package.")
			elif line.startswith('analyzable: '):
				analyzable = int(line.split(None, 1)[1])
			elif line.startswith('text-src: '):
				self.text_src = line.split(None, 1)[1]
			elif line.startswith('extra-files:'):
				self.extra_svm_files = line.split('\t')[1:]
			elif line.startswith('model-id: '):
				self.model_id = line.split(None, 1)[1]
			else:
				raise ValueError("Unexpected argument " + str(line))

		if not analyzable:
			self.predicted_y = [line.strip() for line in fin]
			fin.close()
			return

		self.predicted_y = []
		self.true_y = []
		self.decvals = []
		for line in fin:
			py, y, line = line.split('\t', 2)

			self.predicted_y += [py]
			self.true_y += [y]
			self.decvals += [list(map(lambda x: float(x), line.split()))]
		fin.close()

	def analyzable(self):
		"""
		Return ``True`` if the result is analyzable; otherwise, return ``False``.
		"""
		if None in [self.decvals, self.predicted_y, self.true_y, self.text_src, self.model_id]:
			return False
		
		if type(self.decvals) is not list or type(self.predicted_y) is not list or type(self.true_y) is not list:
			return False

		return True

	def __str__(self):
		if self.analyzable():
			return 'analyzable result (size = {0}): (data: {1}, accuracy: {2})'.format(len(self.predicted_y), self.text_src, self.get_accuracy())
		else:
			return 'unanalyzable result: {0}'.format(self.predicted_y)

class TextModel(object):
	"""
	:class:`TextModel` is a high-level text model. It consists of a 
	:class:`libshorttext.converter.Text2svmConverter` instance and a 
	:class:`learner.LearnerModel` instance.
	:class:`TextModel` is portable and can be saved to or loaded from a 
	directory. It is mainly used to predict short texts by 
	:func:`predict_text` and :func:`predict_single_text`.

	There are two usages of the constructor. For the first one, users can 
	construct a :class:`TextModel` instance from an existing model directory by
	specifying the directory path:

		>>> m = TextModel('model_path')

	For the other one, users can generate a new :class:`learner.TextModel` 
	object with a :class:`libshorttext.converter.Text2svmConverter` instance and
	a :class:`learner.LearnerModel` instance.

		>>> m = TextModel(text2svmConverter, learnerModel)

	In the second usage, both *text2svmConverter* and *learnerModel* can be 
	``None`` or ignored. Therefore, users can create an empty 
	:class:`TextModel`:

		>>> m = TextModel()

	:class:`TextModel` generates an `id` for each model. The `id` will be the same 
	if users load the whole model by the first usage: ``TextModel('model_path')``. 
	However, :class:`TextModel` creates a new `id` if the second usage --
	``TextModel(text2svmConverter, learnerModel)`` -- is used. Therefore, even with
	the same ``text2svmConverter`` and ``learnerModel``, the created :class:`TextModel`
	instances are `different`. Analysis module :mod:`libshorttext.analyzer` uses the 
	`id` to check the consistency of the model and the prediction results. A different `id` 
	may introduce some warning messages, so to reload an existing model, users should 
	use the first usage instead of creating a new model using the same parameters.

	:class:`TextModel` can also be generated by :func:`train_text` or 
	:func:`train_converted_text`. Please refer to :func:`train_text` and 
	:func:`train_converted_text` for the usage.
	"""

	def __init__(self, arg1 = None, arg2 = None):
		if arg2 is None and isinstance(arg1, str):
			self.load(arg1)
			return

		if arg1 is None or isinstance(arg1, Text2svmConverter):
			self.text_converter = arg1
		if arg2 is None or isinstance(arg2, LearnerModel):
			self.svm_model = arg2
	
		# generate an id for TextModel
		from hashlib import sha1
		from time import time

		self._hashcode = sha1('{0} {1:.16g}'.format(id(self), time()).encode('ASCII')).hexdigest()

	def __str__(self):
		return 'TextModel instance ({0}, {1})'.format(self.text_converter, self.svm_model)

	def get_labels(self):
		"""
		Return a list of labels in the model, which should be the labels
		in the training data.
		"""
		return [self.text_converter.getClassName(k) for k in self.svm_model.get_labels()]

	def load(self, model_name):
		""" 
		Load the contents from a :class:`TextModel` directory. The 
		following two code segments are equivalent.
			
		>>> m = TextModel('model_path')

		>>> m = TextModel()
		>>> m.load('model_path')
		"""

		try:
			fin = open(model_name + '/id', 'r')
			self._hashcode = fin.readline().strip()
			fin.close()
		except IOError as ex:
			raise ValueError("The given model is invalid.")
		
		self.text_converter = Text2svmConverter().load(model_name + '/converter')
		self.svm_model = LearnerModel(model_name + '/learner')
		

	def save(self, model_name, force=False):
		"""
		Save the model to a directory. If *force* is set to ``True``, 
		existing directory will be overwritten; otherwise, 
		:class:`IOError` will be raised.
		"""
		if self.svm_model == None:
			raise Exception('This model can not be saved because svm model is not given.')
		
		if path.exists(model_name) and force:
			shutil.rmtree(model_name)

		try:
			os.mkdir(model_name)
		except OSError as e:
			raise OSError(e, 'Please use force option to overwrite the existing files.')

		self.text_converter.save(model_name + '/converter')
		self.svm_model.save(model_name + '/learner', force)
		
		fout = open(model_name + '/id', 'w')
		fout.write(self._hashcode)
		fout.close()


	def get_weight(self, xi, labels = None, extra_svm_feats = []):
		"""
		Return the weights of the model. It only returns the weights
		corresponding to the features of *xi* and the specified *labels*.
		
		*xi* is a :class:`str` of text. It can also be a LIBSVM python
		interface instance. Refer to LIBSVM python interface document
		for more details.

		*labels* is a list of :class:`str` representing the labels.
		If *labels* is ``None``, all labels in the model are considered.

		The returned value is a triple: *features*, *weights*, and
		*labels*. *features* is a list of features extracted from *xi*.
		*labels* is a list of labels. If the input argument *labels* is
		given, the input *labels* will be returned. *weights* is a list
		of lists. The length of *weights* is the number of features,
		and the length of each list in *weights* is the number of 
		labels. The ordering of weights depends on the ordering of
		*features* and *labels*. See the following example.

			>>> features, weights, labels = get_weight('a sentence', ['label 1', 'label 2'])

		If the returned features and weights are:

			>>> print(features, labels)
			['sentence', 'a'] ['label 1', 'label 2']

		then the weight of feature ``'a'`` for ``'label 1'`` can 
		be obtained as follows.

			>>> weights_j = weights[1] # the weights of 'a'
			>>> weights_j_k = weights_j[0] # the weight of 'label 1'

		If the extra features are included when training the model,
		*extra_svm_feats* can be given. *extra_svm_feats* is a 
		class:`list` of class:`list`. The size should be zero or
		the same as the extra svm files in training phase.
		"""


		if self.svm_model == None:
			raise Exception('This model is not usable because svm model is not given.')

		if labels is None:
			labels = self.get_labels()
		elif isinstance(labels, str):
			labels = [labels]
		elif not isinstance(labels, list):
			raise TypeError('labels should be a list of classes.')


		if isinstance(xi, str):
			xi = self.text_converter.toSVM(xi, extra_svm_feats = extra_svm_feats)
		elif extra_svm_feats != 0:
			sys.stderr.write('Warning: extra_svm_feats ignored')
		
		if isinstance(xi, list):
			xi = dict(filter(lambda i: i[1] is not 0, enumerate(xi, 1)))
		elif not isinstance(xi, dict):
			raise TypeError('xi should be a sentence or a LIBSVM instance.')

		numLabels = [self.text_converter.getClassIdx(k) for k in labels]
		not_found_labels = [labels[k] 
				for k in filter(lambda lb: numLabels[lb] is None, range(len(numLabels)))]
		if len(not_found_labels) > 0:
			raise KeyError('The following labels do not exist: ' + ','.join(not_found_labels))
		
		weights = [ [self.svm_model.get_weight(j, k) for k in numLabels] 
				for j in xi]
		
		features = [self.text_converter.get_fidx2tok(j) for j in xi]

		return (features, weights, labels)


def train_converted_text(svm_file, text_converter, grid_arguments = '0', feature_arguments = '', train_arguments = ''):
	"""
	Return a :class:`TextModel` generated from the given LIBSVM-format data.
	*svm_file* is the path of the LIBSVM-format data. The second argument 
	*text_converter* is a :class:`libshorttext.converter.Text2svmConverter`
	instance, which was used before to convert texts to the LIBSVM-format data *svm_file*.

	Refer to :func:`train_text` for details of arguments 
	*grid_arguments*, *feature_arguments*, and *train_arguments*.

	:func:`train_converted_text` assumes that *svm_file* is generated
	from the *text_converter*, so *text_converter* cannot be omitted. If 
	users only need to train a LIBSVM-format data and do not know the 
	preprocessing procedure, :mod:`libshorttext.classifier.learner` module 
	should be used. Refer to :ref:`learner`.

	Note that :func:`train_text` is implemented by calling this function.
	The following two segments of codes are basically equivalent.
	
		>>> from libshorttext.classifier import *
		>>> 
		>>> m, svm_file = train_text('train_file', 'svm_file')

	The above codes can be replaced by using the following codes.
	
		>>> from libshorttext.classifier import *
		>>> from libshorttext.converter import *
		>>> 
		>>> converter = Text2svmConverter()
		>>> svm_file = 'svm_file'
		>>> convert_text('train_file', converter, svm_file)
		>>> m = train_converted_text(svm_file, converter)
	
	The latter one is useful when users want to modify *converter*
	before training. For example, they can replace the default tokenizer of 
	:class:`libshorttext.converter.Text2svmConverter` to a user-defined 
	function before calling :func:`train_converted_text`. Refer to 
	:ref:`CustomizedPreprocessing` for more details of extending 
	:class:`libshorttext.converter.Text2svmConverter`.
	"""

	train_arguments = '-s 4 ' + train_arguments

	if grid_arguments != '0' and grid_arguments != 0:
		default_grid_arguments = '-svmtrain {0} -log2g null -log2c -6,6,2 '.format(path.dirname(__file__) + '/learner/learner_impl.py')
		if grid_arguments == '1' or grid_arguments == 1:
			grid_arguments = default_grid_arguments
		else:
			grid_arguments = default_grid_arguments + grid_arguments
		parameters = find_parameters(svm_file, grid_arguments + ' ' + train_arguments + ' ' + feature_arguments)[1]
		train_arguments = train_arguments + ' -c ' + str(parameters["c"])
	
	m = train(svm_file, feature_arguments, train_arguments)

	return TextModel(text_converter, m)


def train_text(text_src, svm_file = None, converter_arguments = '', grid_arguments = '0', feature_arguments = '', train_arguments = '', extra_svm_files = []):
	"""
	Return a tuple of :class:`TextModel` and :class:`str` instances, where
	the :class:`str` is the file name of the preprocessed LIBSVM-format 
	data.

	*text_src* is the training data path. The data is in text format.
	
	*svm_file* is the path where LIBSVM-format data will be generated. If 
	this value is ``None`` or omitted, the generated LIBSVM-format data
	will be in the working directory with the same file name as *text_src* 
	except the file extension `.svm` is appended. Otherwise, this value is 
	the second returned value.

	*converter_arguments* is the arguments for preprocessor. Refer to
	:func:`libshorttext.converter.convert_text`.

	*grid_arguments* is the arguments passed to :mod:`grid`. Please refer
	to LIBSVM document for the usage. If the *grid_arguments* is set to any
	value other than ``'0'`` or ``0``, this function will apply grid search
	first, and use the best parameter *C* in the training phase. Refer the 
	LIBLINEAR document for more information of the parameter *C*.

	*feat_arguments* is the arguments for feature processing. Refer to
	:ref:`learner_param`.

	*train_arguments* is the LIBLINEAR arguments for training. Refer to
	LIBLINEAR documents. Note that the default solver is ``'-s 4'``.
	
	*extra_svm_files* is a list of user-defined LIBSVM-format file paths 
	to be included during the training. Each file should have the same
	number of instances as *text_src*.

	.. note::

		If *grid_arguments* is not ``'0'`` and ``'-c C'`` option is
		also given in the *train_arguments*, :class:`ValueError` will be
		raised.


	Refer to :func:`train_converted_text` for the comparison between 
	:func:`train_text` and :func:`train_converted_text`.
	"""
	name = path.split(text_src)[1]
	svm_file = svm_file or name + '.svm'

	text_converter = Text2svmConverter(converter_arguments)
	convert_text(text_src, text_converter, svm_file)
	text_converter.merge_svm_files(svm_file, extra_svm_files)

	m = train_converted_text(svm_file, text_converter, grid_arguments, feature_arguments, train_arguments)
	return (m, svm_file)


def predict_text(text_src, text_model, svm_file=None, predict_arguments='', extra_svm_files = []):
	"""
	Return an `analyzable` :class:`PredictionResult` instance. (Refer to
	:class:`PredictionResult`.) The arguments *text_src* and *svm_file* are 
	similar to those of :func:`train_text`, but for returned values
	:func:`predict_text` returns a :class:`PredictionResult` instance instead 
	of a :class:`tuple`. The generated LIBSVM-data path is included in 
	:class:`PredictionResult` as a member value. (Refer to 
	:class:`PredictionResult.svm_file`.) The size of *extra_svm_files* 
	should be zero or the same as the extra svm files of training. Extra 
	svm files are in LIBSVM-format and have the same numbers of instances
	as *text_src*.


	Example:

		>>> from libshorttext.classifier import *
		>>> textModel = TextModel('model_path')
		>>> predictionResult = predict_text('train_file', textModel)
		>>> print(predictionResult.predicted_y)
		['Label 1', 'Label 2', 'Label 1']

	Similar to *train_arguments* in :func:`train_context`, 
	*predict_arguments* is the arguments for LIBLINEAR predict. Refer to
	LIBLINEAR documents for more details.

	Although there are :func:`train_text` and :func:`train_converted_text`
	for training, there is **no** :func:`predict_converted_text`. Use 
	:func:`libshorttext.classifier.learner.predict` instead for converted data.
	"""
	
	name = path.split(text_src)[1]
	svm_file = svm_file or name + '.svm'

	text_converter = text_model.text_converter
	convert_text(text_src, text_converter, svm_file)
	text_converter.merge_svm_files(svm_file, extra_svm_files)

	predicted_y, acc, dec_vals, true_y = predict(svm_file, text_model.svm_model, predict_arguments)

	predicted_y = [text_model.text_converter.getClassName(int(y))
			for y in predicted_y]
	true_y = [text_model.text_converter.getClassName(int(y))
			for y in true_y]


	return PredictionResult(text_src, text_model._hashcode, true_y, predicted_y, dec_vals, svm_file, text_model.get_labels(), extra_svm_files)


def predict_single_text(text, text_model, predict_arguments = '', extra_svm_feats = []):
	"""
	Return an `unanalyzable` :class:`PredictionResult` instance. (Refer to
	:class:`PredictionResult`.) 

	*text* is a short text in :class:`str`. *text* can also be a LIBSVM list
	or dictionary. Refer to LIBSVM python interface document for more 
	details.

	*extra_svm_feats* is a list of extra features. The size is zero or the
	same as the extra features of training. Extra features are class:`dict`.
	
	.. note::
		*extra_svm_feats* are ignored if *text* is already a LIBSVM
		list or dictionary.
	
	.. note::
		
		In this version, there is no predict option and argument 
		*predict_arguments* will be ignored.

	.. note::
		
		This function is designed to analyze the result of some specific
		short texts. It has a severe efficiency issue. If many instances
		need to be predicted, they should be stored in a file
		and predicted by :func:`predict_text` or 
		:func:`libshorttext.classifier.learner.predict`.

	"""
	if not isinstance(text_model, TextModel):
		raise TypeError('argument 1 should be TextModel')

	if text_model.svm_model == None:
		raise Exception('This model is not usable because svm model is not given')
	
	if isinstance(text, str):
		text = text_model.text_converter.toSVM(text, extra_svm_feats = extra_svm_feats)
	elif isinstance(text, (list, dict)):
		if extra_svm_feats:
			sys.stderr.write('Warning: extra_svm_feats ignored')
	else:
		raise TypeError('The argument should be plain text or LIBSVM-format data.')

	y, dec = predict_one(text, text_model.svm_model)

	y = text_model.text_converter.getClassName(int(y))
	labels = [text_model.text_converter.getClassName(k) for k in text_model.svm_model.label[:text_model.svm_model.nr_class]]

	return PredictionResult(predicted_y = y, decvals = dec[:text_model.svm_model.nr_class], labels = labels)
	#return (y, dec[:text_model.svm_model.nr_class], labels)
