local utf8 = require "icu.utf8"
local regex = require "icu.regex"
local ustring = require "icu.ustring"

local definedStopwords = require "tokenizer.stopwords"
local definedPatterns = {}

module((...), package.seeall)

--forward declarations
local removeTxtFn
local normalize
local addStopWords
local splitWords
local makeURLPattern

--[[ Possible options
{
	lowercase = true,
	normalizeWords = true,
	extraPatternMatch = true,
	countWords = true,
	definedStopWords = {"html", "general"},
	customStopWords = "the in or and",
	includeDefinedPatterns = {"url", "hashtag"},
	excludeDefinedPatterns = {"mailaddress"},
	includeCustomPatterns = {"%a+@gmail"},
	excludeCustomPatterns = {"#%a+"},
}
]]

function processBundle(text, optionsBundle)
	local result = {}
	for k, v in pairs (optionsBundle) do
		result[k] = process(text, v)
	end
	return result
end

-- Process text with the option supplied.

-- First we take in account the defined patterns to include,
-- next we process the custom patterns to include,
-- then we process the defined patterns to exclude from tokenization,
-- and next we exclude custom patterns.

-- Every pattern that match is removed from text
-- so it won't be tokenized later as individual words.

-- Finally the result set is filtered by custom and predefined stopwords lists.

function process(text, options)
	options = options or {}
	options.lowercase = options.lowercase == nil and true or options.lowercase
	options.normalizeWords = options.normalizeWords == nil and false or options.normalizeWords
	options.extraPatternMatch = options.extraPatternMatch == nil and false or options.extraPatternMatch
	options.countWords = options.countWords == nil and true or options.countWords
	
	if options.lowercase then
		text = icu.utf8.lower(text)
	end
	if options.normalizeWords then
		text = normalize(text)
	end
	
	local stopwordSet = {}
	
	if options.customStopWords then
		addStopWords(stopwordSet, options.customStopWords, options)
	end
	
	if options.definedStopWords then
		for _, definedSet in ipairs(options.definedStopWords) do
			assert(definedStopwords[definedSet], "defined definedStopWords "..definedSet.." does not exist")
			addStopWords(stopwordSet, definedStopwords[definedSet], options)
		end
	end
	
	local result = {}
	
	local function addWord(word)
		if options.countWords then
			result[word] = (result[word] or 0) + 1
		else
			result[word] = true
		end
	end
	
	if options.includeDefinedPatterns then
		for _, pattern in ipairs(options.includeDefinedPatterns) do
			assert(definedPatterns[pattern], "defined pattern "..pattern.." does not exist")
			
			local function matchFn(match)
				addWord(match)
				if options.extraPatternMatch and type(definedPatterns[pattern].extra) == "table" then
					for _, extraPattern in ipairs(definedPatterns[pattern].extra) do
						for _, extraMatch in ipairs{match:match(extraPattern)} do
							--in some cases principal match is equal to extra match
							--and we don't want to add the same word twice
							if extraMatch ~= match then
								addWord(extraMatch)
							end
						end
					end
				end
				return ""
			end
			
			if definedPatterns[pattern].regex then
				local compiledRegex = regex.compile(definedPatterns[pattern].principal)
				text = ustring.encode(icu.regex.replace (compiledRegex, ustring(text), function(matchObject)
					return matchFn(ustring.encode(matchObject.value))
				end))
			else
				text = text:gsub(definedPatterns[pattern].principal, matchFn)
			end
		end
	end
	
	if options.includeCustomPatterns then
		for _, pattern in ipairs(options.includeCustomPatterns) do
			text = text:gsub(pattern, function (match)
				addWord(match)
				return ""
			end)
		end
	end
	
	if options.excludeDefinedPatterns then
		for _, pattern in ipairs(options.excludeDefinedPatterns) do
			assert(definedPatterns[pattern], "defined pattern "..pattern.." does not exist")
			
			if definedPatterns[pattern].regex then
				local compiledRegex = regex.compile(definedPatterns[pattern].principal)
				text = ustring.encode(icu.regex.replace (compiledRegex, ustring(text), removeTxtFn))
			else
				text = text:gsub(definedPatterns[pattern].principal, removeTxtFn)
			end
		end
	end
	
	if options.excludeCustomPatterns then
		for _, pattern in ipairs(options.excludeCustomPatterns) do
			text = text:gsub(pattern, removeTxtFn)
		end
	end
	
	local words = splitWords(text)
	for _, word in ipairs(words) do
		if not stopwordSet[word] then
			addWord(word)
		end
	end
	
	return result
end

--
-- Internal functions
--

function removeTxtFn ()
	return ""
end

function addStopWords(set, stopwords, options)
	if options.lowercase then
		stopwords = icu.utf8.lower(stopwords)
	end
	if options.normalizeWords then
		stopwords = normalize(stopwords)
	end
	
	local words = splitWords(stopwords)
	for _, word in ipairs(words) do
		set[word] = true
	end
end


function normalize (word)
	local map = {
		["á"]="a", ["é"]="e", ["í"]="i", ["ó"]="o", ["ú"]="u",
		["à"]="a", ["è"]="e", ["ì"]="i", ["ò"]="o",	["ù"]="u",
		["ä"]="a", ["ë"]="e", ["ï"]="i", ["ö"]="o", ["ü"]="u",
		["â"]="a", ["ê"]="e", ["î"]="i", ["ô"]="o",	["û"]="u",
		["Á"]="A", ["É"]="E", ["Í"]="I", ["Ó"]="O", ["Ú"]="U",
		["À"]="A", ["È"]="E", ["Ì"]="I", ["Ò"]="O",	["Ù"]="U",
		["Ä"]="A", ["Ë"]="E", ["Ï"]="I", ["Ö"]="O", ["Ü"]="U",
		["Â"]="A", ["Ê"]="E", ["Î"]="I", ["Ô"]="O",	["Û"]="U",
	}
	local normalized = utf8.gsub(word, "%!a", map)
	
	return normalized
end


function splitWords (str)
	local t = {}
	for word in utf8.gmatch(str, "%!a+") do
		table.insert(t, word)
	end
	return t
end

--
-- Defined patterns
--

local function makeURLPattern()
	local exclude = {}
	for i = 1, 31 do
		table.insert(exclude, i)
	end
	for i = 127, 255 do
		table.insert(exclude, i)
	end
	
	return '%a+://[^%z "\'<>{}|\^~%[%]`'..string.char(unpack(exclude))..']+'
end

definedPatterns.url = {
	principal = makeURLPattern(),
	extra = {
		'(%a+://(%a+%.([^/]+)))', --matchs "http://www.inconcertcc.com", "www.inconcertcc.com", "inconcertcc.com"
	}
}

definedPatterns.mailaddress = {
	regex = true, --only principal expression is considered a regex
	principal = [[(".+"\s*)?\<?[\w\._-]+@[\w\._]+\>?]], --no silver bullet here... it will not match 'Pepe Gonzales <pepeg@gmail.com>'
	extra = {
		'<?(([%w%._%-]+)@[%w%._]+)',
		'"(.+)"',
	}
}

definedPatterns.hashtag = {
	principal = '#%a[%w_]*',
	extra = {
		'#(%a[%w_]*)',
	}
}

definedPatterns.twitterUsername = {
	principal = '[^%w%._%-]@%a[%w_]*',
	extra = {
		'[^%w%._%-]@(%a[%w_]*)',
	},
}

definedPatterns.guid = {
	regex = true, --only principal expression is considered a regex
	principal = [[[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}]],
	extra = {
		"(.+)%-(.+)%-(.+)%-(.+)%-(.+)"
	},
}

definedPatterns.question = {
	principal = '%w+%?+',
	extra = {
		'(%w+)%?+',
	},
}

definedPatterns.exclamation = {
	principal = '%w+%!+',
	extra = {
		'(%w+)%!+'
	},
}

-- Regexp taken from http://www.regular-expressions.info/floatingpoint.html
definedPatterns.numbers = {
	regex = true,
	principal = '[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?'
}
