Module:Util/schema/validate

---

local h = {}

local utilsError = require("Module:UtilsError")
local utilsTable = require("Module:UtilsTable")
local utilsValidate = require("Module:UtilsValidate")
local util = {
	markup = {
		code = require("Module:Util/markup/code")
	}
}

local metaSchema = require("Module:Util/schemas/validate/Documentation/Spec").Schemas()

local SYMBOLS = {
	optional = "[%s]",
	required = "%s!",
	default = "%s=%s",
	array = "{%s}",
	map = "<%s, %s>",
	oneOf = "%s|%s",
	allOf = "%s&%s",
	combinationGroup = "(%s)",
}

local TYPES = {
	oneOf = "oneOf",
	array = "array",
	record = "record",
	map = "map"
}

function validate(schema, schemaName, data, dataName)
	-- first validate the schema itself
	h.collectReferences(metaSchema)
	local schemaErrors = h.validate(metaSchema.values, schema, schemaName)
	if #schemaErrors > 0 then
		utilsError.warn(string.format("Schema <code>%s</code> is invalid.", schemaName))
	end
	-- then validate the data
	h.collectReferences(schema)
	local errors = h.validate(schema, data, dataName)
	if #errors > 0 then
		utilsError.warn(string.format("<code>%s</code> is invalid according to schema <code>%s</code>", dataName, schemaName))
		return errors
	end
end

function h.collectReferences(schema)
	h.references = {}
	h.references["#"] = schema
	for k, v in pairs(schema.definitions or {}) do
		h.references["#/definitions/" .. k] = v
		h.collectIdReferences(v)
	end
	h.collectIdReferences(schema)
end
function h.collectIdReferences(schema)
	h.walkSchema(function(schemaNode)
		if schemaNode._id then
			h.references[schemaNode._id] = schemaNode
		end
	end, schema)
end

function h.resolveReference(schemaNode)
	if schemaNode._ref then
		local referenceNode = h.references[schemaNode._ref]
		if not referenceNode then
			mw.addWarning(string.format("<code>%s</code> not found", mw.text.nowiki(schemaNode._ref)))
		else
			local resolvedSchema = utilsTable.merge({}, h.references[schemaNode._ref], schemaNode)
			schemaNode = utilsTable.merge({}, schemaNode, resolvedSchema)
			schemaNode._ref = nil
		end
	end
	return schemaNode
end

function h.walkSchema(fn, schemaNode, path)
	path = path or {}
	local continue = fn(schemaNode, path)
	if continue == false then
		return
	end
	if schemaNode.items then
		h.walkSchema(fn, schemaNode.items, utilsTable.concat(path, "items"))
	end
	if schemaNode.keys then
		h.walkSchema(fn, schemaNode.keys, utilsTable.concat(path, "keys"))
	end
	if schemaNode.values then
		h.walkSchema(fn, schemaNode.values, utilsTable.concat(path, "values"))
	end
	if schemaNode.properties then
		for i, v in ipairs(schemaNode.properties) do
			local keyPath = utilsTable.concat(path, "properties", v.name)
			h.walkSchema(fn, v, keyPath)
		end
	end
	if schemaNode.oneOf then
		for k, v in pairs(schemaNode.oneOf) do
			h.walkSchema(fn, v, utilsTable.concat(path, "oneOf"))
		end
	end
	if schemaNode.allOf then
		for i, v in ipairs(schemaNode.allOf) do
			h.walkSchema(fn, v, utilsTable.concat(path, "allOf"))
		end
	end
	if schemaNode.definitions then
		for k, v in pairs(schemaNode.definitions) do
			h.walkSchema(fn, v, utilsTable.concat(path, "definitions", k))
		end
	end
end

-- This is to ensure that the documentation for recursive refs is shown only once.
function h.minRefDepths(schema)
	h.minDepthNode = {}
	local minDepths = {}
	h.walkSchema(function(schemaNode, path)
		if schemaNode._ref then
			minDepths[schemaNode._ref] = math.min(minDepths[schemaNode._ref] or 9000, #path)
			if #path == minDepths[schemaNode._ref] then
				h.minDepthNode[schemaNode._ref] = schemaNode
			end
		end
	end, schema)
end
function h.hasRefs(schema)
	local hasRefs = false
	h.walkSchema(function(schemaNode)
		if schemaNode._ref then
			hasRefs = true
			return false
		end
	end, schema)
	return hasRefs
end
function h.showSubkeys(schemaNode)
	local ref = schemaNode._ref
	if schemaNode._hideSubkeys then
		return false
	elseif not ref then
		return true
	elseif not string.find(ref, "^#/definitions") then
		return false
	else
		return h.minDepthNode[ref] == schemaNode or not h.hasRefs(h.references[ref])
	end
end

function h.getTypeDefs(schemaName, schema, formattingFn, parentSchema, isSubschema)
	local typeLabel = schema.typeLabel
	local showSubkeys = true
	if schema._ref then
		typeLabel = typeLabel or string.gsub(schema._ref, "#/definitions/", "")
		typeLabel = typeLabel or string.gsub(typeLabel, "#", "")
		showSubkeys = h.showSubkeys(schema)
	end
	schema = h.resolveReference(schema)
	
	local rawType = schema.type
	local symbolicType
	local subkeys
	if showSubkeys then
		if schema.type == TYPES.record then
			for _, prop in ipairs(schema.properties) do
				subkeys = subkeys or {}
				local propDef = h.getTypeDefs(prop.name, prop, formattingFn, schema)
				table.insert(subkeys, propDef)
			end
		end
		if schema.type == TYPES.array then
			local subtypeKeys, subtype = h.getTypeDefs(nil, schema.items, formattingFn, schema)
			if #subtypeKeys > 0 then
				subkeys = subtypeKeys
			end
			symbolicType = string.format(SYMBOLS.array, subtype)
		end
		if schema.type == TYPES.map then
			local _, keyType = h.getTypeDefs(nil, schema.keys, formattingFn, schema)
			local valueDef, valueType = h.getTypeDefs(nil, schema.values, formattingFn, schema)
			symbolicType = string.format(SYMBOLS.map, keyType, valueType)
			subkeys = valueDef
		end
		if schema.oneOf then
			subkeys = subkeys or {}
			subkeys.oneOf = subkeys.oneOf or {}
			local subtypes = {}
			local i = 1
			for k, subschema in pairs(schema.oneOf) do
				if type(k) == "string" then
					subschema.typeLabel = k
				end
				local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema, true)
				if string.find(subtype, "|") or string.find(subtype, "&") then
					subtype = string.format(SYMBOLS.combinationGroup, subtype)
				end
				table.insert(subtypes, subtype)
				subkeys.oneOf[i] = keys
				i = i + 1
			end
			symbolicType = subtypes[1]
			for i, subtype in ipairs(utilsTable.tail(subtypes)) do
				symbolicType = string.format(SYMBOLS.oneOf, symbolicType, subtype)
			end
		end
		if schema.allOf then
			subkeys = subkeys or {}
			subkeys.allOf = {}
			local subtypes = {}
			for i, subschema in ipairs(schema.allOf) do
				local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema)
				if string.find(subtype, "|") or string.find(subtype, "&") then
					subtype = string.format(SYMBOLS.combinationGroup, subtype)
				end
				table.insert(subtypes, subtype)
				subkeys.allOf[i] = keys
			end
			subtypes = utilsTable.unique(subtypes)
			symbolicType = subtypes[1]
			for i, subtype in ipairs(utilsTable.tail(subtypes)) do
				symbolicType = string.format(SYMBOLS.allOf, symbolicType, subtype)
			end
		end
	end
	symbolicType = symbolicType or rawType or typeLabel
	
	local parentType = parentSchema and parentSchema.type
	if parentType == "array" and typeLabel then
		symbolicType = typeLabel
	end
	
	local key = schemaName 
	if schema.default then
		key = key and string.format(SYMBOLS.default, schemaName, tostring(schema.default))
	end
	if parentSchema == nil or not (parentSchema.allOf or parentSchema.oneOf or parentType == TYPES.array or parentType == TYPES.map) then  -- otherwise leads to nonsense like [[{[string]}]|[[string]]], instead of [{string}|string]
		if schema.required then
			symbolicType = string.format(SYMBOLS.required, symbolicType)
		else
			symbolicType = string.format(SYMBOLS.optional, symbolicType)
			key = key and string.format(SYMBOLS.optional, key)
		end
	end
	
	local formattedDef = formattingFn({
		key = key,
		subkeys = subkeys,
		rawType = rawType,
		typeLabel = typeLabel,
		symbolicType = symbolicType,
		desc = schema.desc,
		parentType = parentType,
		isSubschema = isSubschema,
	})
	return formattedDef, symbolicType
end

function h.validate(schemaNode, data, dataName, dataPath, parentSchema, quiet)
	dataPath = dataPath or {}
	local value = utilsTable.property(data, dataPath)
	local errPath = dataName .. utilsTable.printPath(dataPath)
	schemaNode = h.resolveReference(schemaNode)
	
	local errors = h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet)
	if #errors > 0 then
		return errors
	end
	
	if schemaNode.allOf and value then
		local subschemaErrors = h.validateSubschemas(schemaNode.allOf, data, dataName, dataPath, schemaNode)
		local invalidSubschemas = utilsTable.keys(subschemaErrors)
		if #invalidSubschemas > 0 then
			invalidSubschemas = utilsTable.map(invalidSubschemas, util.markup.code)
			invalidSubschemas = mw.text.listToText(invalidSubschemas)
			local msg = string.format("<code>%s</code> does not match <code>allOf</code> sub-schemas %s", errPath, invalidSubschemas)
			if not quiet then
				utilsError.warn(msg, { traceBack = false })
				h.logSubschemaErrors(subschemaErrors)
			end
			table.insert(errors, {
				path = errPath,
				msg = msg,
				errors = subschemaErrors
			})
		end
	end
	if schemaNode.oneOf and value then
		local subschemaErrors, validSubschemas = h.validateSubschemas(schemaNode.oneOf, data, dataName, dataPath, schemaNode, parentSchema)
		local invalidSubschemas = utilsTable.keys(subschemaErrors)
		if #validSubschemas == 0 then
			local msg = string.format("<code>%s</code> does not match any <code>oneOf</code> sub-schemas.", errPath)
			if not quiet then
				utilsError.warn(msg, { traceBack = false })
				h.logSubschemaErrors(subschemaErrors)
			end
			table.insert(errors, {
				path = errPath,
				msg = msg,
				errors = subschemaErrors
			})
		end
		if #validSubschemas > 1 then
			validSubschemas = utilsTable.map(validSubschemas, util.markup.code)
			validSubschemas = mw.text.listToText(validSubschemas)
			local msg = string.format("<code>%s</code> matches <code>oneOf</code> sub-schemas %s, but must match only one.", errPath, validSubschemas)
			if not quiet then
				utilsError.warn(msg, { traceBack = false })
			end
			table.insert(errors, {
				path = errPath,
				msg = msg,
			})
		end
	end
	
	if schemaNode.properties and value then
		for _, propSchema in pairs(schemaNode.properties) do
			local keyPath = utilsTable.concat(dataPath, propSchema.name)
			errors = utilsTable.concat(errors, h.validate(propSchema, data, dataName, keyPath, schemaNode, quiet))
		end
		if not schemaNode.additionalProperties and not (parentSchema and parentSchema.allOf) then
			local schemaProps = schemaNode.properties
			if parentSchema and parentSchema.allOfProps then
				schemaProps = utilsTable.concat(schemaProps, parentSchema.allOfProps)
			end
			local schemaProps = utilsTable.map(schemaProps, "name")
			local dataProps = utilsTable.keys(value)
			local undefinedProps = utilsTable.difference(dataProps, schemaProps)
			if #undefinedProps > 0 then
				undefinedProps = mw.text.listToText(utilsTable.map(undefinedProps, util.markup.code))
				local msg = string.format("Record <code>%s</code> has undefined properties: %s", errPath, undefinedProps)
				if not quiet then
					utilsError.warn(msg, { traceBack = false })
				end
				table.insert(errors, {
					path = errPath,
					msg = msg,
				})
			end
		end
	end
	if schemaNode.items and value then
		local props = utilsTable.stringKeys(value)
		if #props > 0 and not (parentSchema and parentSchema.allOf) then
			local msg = string.format("<code>%s</code> is supposed to be an array only, but it has string keys: %s", errPath, utilsTable.print(props))
			if not quiet then
				utilsError.warn(msg, { traceBack = false })
			end
			table.insert(errors, {
				path = errPath,
				msg = msg,
			})
		end
		for i, item in ipairs(value) do
			local itemPath = utilsTable.concat(dataPath, i)
			errors = utilsTable.concat(errors, h.validate(schemaNode.items, data, dataName, itemPath, schemaNode, quiet))
		end
	end
	if schemaNode.keys and schemaNode.values and value then
		for k, v in pairs(value) do
			local keyPath = utilsTable.concat(dataPath, string.format('["%s"]', k))
			errors = utilsTable.concat(errors, h.validatePrimitive(schemaNode.keys, k, dataName, dataPath, quiet, true))
			errors = utilsTable.concat(errors, h.validate(schemaNode.values, data, dataName, keyPath, schemaNode, quiet))
		end
	end
	
	return errors
end

function h.validateSubschemas(subschemas, data, dataName, dataPath, schemaNode, parentSchema)
	local errors = {}
	local valids = {}
	for k, subschema in pairs(subschemas) do
		local key = subschema._ref and string.gsub(subschema._ref, "#/definitions/", "") or k
		if parentSchema and parentSchema.allOf then
			local commonProps = {}
			for _, v in ipairs(parentSchema.allOf) do
				commonProps = utilsTable.concat(commonProps, v.properties)
			end
			schemaNode.allOfProps = commonProps
		end
		local err = h.validate(subschema, data, dataName, dataPath, schemaNode, true)
		if #err > 0 then
			errors[key] = err
		else
			table.insert(valids, key)
		end
	end
	return errors, valids
end
function h.logSubschemaErrors(subschemaErrors, path)
	for schemaKey, schemaErrors in pairs(subschemaErrors) do
		local subpath = utilsTable.concat(path or {}, '["' .. schemaKey .. '"]')
		local indent = string.rep(":", #subpath)
		for _, err in pairs(schemaErrors) do
			local msg = string.format("<code>%s</code>: %s", utilsTable.printPath(subpath), err.msg)
			utilsError.warn(indent .. msg, { traceBack = false })
			if err.errors then
				h.logSubschemaErrors(err.errors, subpath)
			end
		end
	end
end

function h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet, isKey)
	local errors = {}
	local addIfError = h.errorCollector(errors, value, dataName, dataPath, quiet, isKey)
	local validatorOptions = {
		quiet = quiet,
		stackTrace = false,
	}
	if schemaNode.type and schemaNode.type ~= "any" then
		local expectedType = h.getLuaType(schemaNode.type)
		addIfError(utilsValidate.type(expectedType))
	end
	if schemaNode.required then
		addIfError(utilsValidate.required)
	end
	if schemaNode.deprecated then
		addIfError(utilsValidate.deprecated)
	end
	if schemaNode.enum then
		addIfError(utilsValidate.enum(schemaNode.enum))
	end
	return errors
end

function h.errorCollector(errorTbl, value, dataName, dataPath, quiet, isKey) 
	return function(validator)
		local errorMessages = validator(value, dataName, dataPath, isKey, {
			quiet = quiet,
			stackTrace = false,
		})
		if type(errorMessages) ~= "table" then --errMsg can either be a single message, or an array of messages (as is the case with enum)
			errorMessages = {errorMessages}
		end
		for _, errMsg in ipairs(errorMessages) do
			table.insert(errorTbl, {
				path = dataName .. utilsTable.printPath(dataPath),
				msg = errMsg,
			})
		end
	end
end

function h.getLuaType(schemaType)
	if schemaType == TYPES.array 
	or schemaType == TYPES.record 
	or schemaType == TYPES.map 
	then
		return "table"
	end
	return schemaType
end

return validate