local utf8 = require "icu.utf8"
module((...), package.seeall)
 
--Loads and return the dictionary from the specified path
--The caller must keep this dictionary in order to use the detect feature
--If path is not specified, then it loads the default dictionary
function LoadDictionary (path, dictionaryParam)

	local dictionary
	if path == nil or path =="" then
		local full = debug.getinfo(1, "S").source:match("^@(.+)")
		path = string.match(full, "^(.*)[/\\]([^/\\]*)$") .. "/languageDetector/dictionary.txt"
	end
	
	if not dictionaryParam or next(dictionaryParam) == nil then
		dictionary = dofile(path)
	else
		dictionary = dictionaryParam
	end
	
	--normalize ocurrences
	for lang, words in pairs (dictionary) do
		local total = 0
		for word,count in pairs (words) do
			total = total + count
		end
		for w, count in pairs (words) do
			words[w] = count / total
		end
	end
	return dictionary	
end

--Try to detect the language of the given text, based in the dictionary supplied
function DetectLanguage (text, dictionary)

	local function splitWords(str)
		local t = {}
		local function aux(word)
			table.insert(t, word)
			return ""
		end
		icu.utf8.gsub(str, "[%!a]+", aux)
		return t
	end

	local function countLanguageWords (words, languageWords)
		
		local count = 0	
		for _, w in ipairs (words) do
			if languageWords[w] then
				count = count + languageWords[w]
			end
		end
		return count
	end

	local lower = utf8.lower (text)
	local words = splitWords (lower or text)
	
	local bestLanguage = ""
	local bestCount = 0
	local totalCount = 0

	for lang, langWords in pairs (dictionary) do
		local count = countLanguageWords (words, langWords)
		totalCount = totalCount + count
		if count > bestCount then
			bestCount = count
			bestLanguage = lang
		end
	end
	
	if bestLanguage == "" then
		return nil
	else
		return bestLanguage, bestCount / totalCount
	end
end
