module(..., package.seeall);
local tokenizer      = require "tokenizer"
local table_helpers  = require "i6index.table_helpers"
local schema_helpers = require "i6index.schema_helpers"
local utf8           = require "icu.utf8"

local function tokenizeContent(text)
	return table_helpers.getKeys(tokenizer.process(text, {
	                     lower_case = true,
	                     normalizeWords = true,
	                     countWords = false,
	                     definedStopWords = {"html"},
	                     includeDefinedPatterns = {"url", "hashtag", "numbers"}
	                    }))
end

local function tokenizeAddresses(addresses)
	text = table.concat(addresses or {}, " ")
	return table_helpers.getKeys(tokenizer.process(text, {
	                     lower_case = true,
	                     normalizeWords = true,
	                     countWords = false,
	                     definedStopWords = {"html", "general"}
	                    }))
end

local function trimElements(field)
	local data = {}
	for _, v in ipairs(field or {}) do table.insert(data, schema_helpers.trim(v)) end
	return data
end

local function tokenizeName(name)
	return table_helpers.getKeys(tokenizer.process(name, {
	                     lower_case = true,
	                     normalizeWords = true,
	                     countWords = false,
	                     }))
end

-- this is the document definition for interactions
local interactionDocument = {
	-- if tokenized = true then ${field}Tokenizer is used to get tokens
	-- if multiValued = true then get${capitalizeFirst(field)} to get tokens

	indexes = {
	    type             = {tokenized = false, multiValued = false},
	    address          = {tokenized = false, multiValued = true},
	    vcc              = {tokenized = false, multiValued = false},
	    contact          = {tokenized = false, multiValued = false},
	    contact_name     = {tokenized = false, multiValued = false},
	    client_id        = {tokenized = false, multiValued = false},
	    disposition_code = {tokenized = false, multiValued = false},
	    content          = {tokenized = true,  multiValued = true},
	    cci              = {tokenized = false, multiValued = true},
	    frontend         = {tokenized = false, multiValued = true},
	    agent            = {tokenized = false, multiValued = true},
	    label            = {tokenized = false, multiValued = true},
	},

	--[[
	    the "composite indexes" are with more than one value, for example if you add a document:
	    {
	        type = "MAIL",
	        agent = "deborah.meltrozzo@fing.edu.uy"
	    }
	    it will insert "MAIL:deborah.meltrozzo@fing.edu.uy" in type_agent index.

	    + compositeIndexLevel determines how many indexes get grouped. if compositeIndexLevel is n
	      then it will be all the combinations of 2, 3 .. n indexes

	    + the combination indexes are sorted alphabetically, if level is 2 then for the next set
	      of fields:
	        type         = {tokenized = false, multiValued = false},
	        address      = {tokenized = false, multiValued = false},
	        vcc          = {tokenized = false, multiValued = false},
	        contact      = {tokenized = false, multiValued = false},
	        contact_name = {tokenized = false, multiValued = false},
	      then the next composite indexes will be created:
	        address-contact
	        address-contact_name
	        address-type
	        address-vcc
	        contact-contact_name
	        contact-type
	        contact-vcc
	        contact_name-type
	        contact_name-vcc
	        type-vcc
	--]]
	compositeIndexLevel = 2,
	forbiddenComposites = {
		vcc = { "address", "agent", "contact", "contact_name", "cci", "frontend", "disposition_code", "label", "type", "client_id", "content" },
		content = {"cci", "frontend", "label", "vcc"},
		client_id = {"cci", "frontend", "label", "vcc"},
		label = { "cci", "frontend" },
		type = { "cci", "frontend" },
	},
--======================================================================
	contentTokenizer = function (v)
	    return tokenizeContent(v)
	end,
	sortField = "date",

	name = "interaction"
};

local contactDocument = {
	indexes = {
	    type     = {tokenized = false, multiValued = false},
	    company  = {tokenized = false, multiValued = false},
	    vcc      = {tokenized = false, multiValued = false},
	    facebook = {tokenized = false, multiValued = true},
	    instagram = {tokenized = false, multiValued = true},
	    youtube = {tokenized = false, multiValued = true},
	    appstore = {tokenized = false, multiValued = true},
	    googleplay = {tokenized = false, multiValued = true},
	    twitter  = {tokenized = false, multiValued = true},
	    im       = {tokenized = false, multiValued = true},
	    email    = {tokenized = false, multiValued = true},
	    phone    = {tokenized = false, multiValued = true},
	    webchat  = {tokenized = false, multiValued = true},
	    default  = {tokenized = false, multiValued = true},
	},

	compositeIndexLevel = 2,
--======================================================================
	-- getter for the multivalued field "default", note that it acts as a sum of all fields.
	getDefault = function(document)
	     local data = {}
	     if schema_helpers.isValidAsValue(document.type)    then table.insert(data, document.type)    end
	     if schema_helpers.isValidAsValue(document.company) then table.insert(data, document.company) end
	     if schema_helpers.isValidAsValue(document.vcc)     then table.insert(data, document.vcc)     end

	     table_helpers.append(data, tokenizeName(document.name))

	     table_helpers.append(data, tokenizeAddresses(document.facebook))
	     table_helpers.append(data, tokenizeAddresses(document.twitter))
	     table_helpers.append(data, tokenizeAddresses(document.im))
	     table_helpers.append(data, tokenizeAddresses(document.email))
	     table_helpers.append(data, tokenizeAddresses(document.phone))
	     table_helpers.append(data, tokenizeAddresses(document.webchat))

	     -- in addition to the tokens of each field the trimmed original version of the fields it's needed
	     -- in order to satisfy queries by the complete address. For example if a document has as mail
	     -- address the value: marko.ramius@redoctober.ru the tokens will be: marko, ramius, redoctober, and ru.
	     -- So querying for any token should work, but if we don't add marko.ramius@redoctober.ru we won't be
	     -- able to satisfy queries issuing the complete address

	     table_helpers.append(data, trimElements(document.facebook))
	     table_helpers.append(data, trimElements(document.twitter))
	     table_helpers.append(data, trimElements(document.im))
	     table_helpers.append(data, trimElements(document.email))
	     table_helpers.append(data, trimElements(document.phone))
	     table_helpers.append(data, trimElements(document.webchat))

	     return data
	 end,
--======================================================================
	sortComparator = "UTF8Type",
	sortField      = "name",
	name           = "contact"
--======================================================================
};

local s = {
	definitions = {},
	keyspace="Indexes"
}

s.definitions[interactionDocument.name] = interactionDocument
s.definitions[contactDocument.name]     = contactDocument

return s

