local p={}
local debuglog = ""

function cleanup(str)
	str = mw.ustring.match(str, "^%s*(%S.-%S)%s*$") or str -- (pass through "" and one character stuff)
	str = mw.ustring.gsub(str,"Module:","") -- remove all "Module:" from these, so they match up
	return str
end

function p.test(frame)
	return mw.text.unstrip(frame:preprocess("{{Special:AllPages|from=Listify/doc|to=Sandbox/Theopolisme|namespace=828}}"))
end

function p.getTable(frame)
    local alltable = p._getAll(frame)
    for i = 1, #alltable do
        alltable[i] = "|-\n|[[Module:" .. alltable[i] .. "|" .. alltable[i] .. "]]"
    end
    return '{| class = "wikitable"\n' .. table.concat(alltable,'\n') .. '\n|}\n'
end

function p._getAll(frame)
    local modules = ""
    local from = ""
    local lastfrom
    local pagelist = {}
    repeat
        local index = mw.text.unstrip(frame:preprocess("{{Special:AllPages|namespace=828|from=" .. from .. "}}"))
        local nextlink = mw.ustring.gmatch(index, "<a.->.-:(.-)</a>")
        if (from ~= "") then
        	nextlink() -- discard the first value; it'll be the same as the from, i.e. the last value added last time
        end
        local link = true
        while true do
            lastlink = link
            link = nextlink()
            if (not link) then break end
            table.insert(pagelist, mw.text.decode(link))
        end
        lastfrom = from
        from = lastlink
    until ((lastfrom == from) or (from == true)) -- I think only the last is needed but I hate debugging loops
    return pagelist
end

function p.getAll(frame)
	return table.concat(p._getAll(frame), "\n") .. debuglog
end

function dictinsert(dict, data, heading) -- passing in a reference to the dictionary, not a copy of it, so whichever is specified gets altered
	local getentry = mw.ustring.gmatch(data, "([^\n\t]+)")
	local entry = " "
	while entry do
		entry = cleanup(entry)
		dict[entry] = heading
	    entry = getentry()
	end
end

function p.main(frame)
	local args = frame.args -- not bothering reading parent frame, this is to be invoked only
    local title = args[1]
    return p._main(title, frame)
end

function p._main(title, frame)
    local titleObject, titleContent
    if title then
    	titleObject = mw.title.new(title)
    	if titleObject then
    		datapage = titleObject:getContent()
	    end
	end
	if not datapage then
		return "Error: data page not found"
    end
    -- OK, we're now looking at a big juicy file with a lot of sections in top level headings surrounded by == ==
    -- Many of these then are subject to commands in [ ] that follow.  The heading name should be remembered to each of these.
    -- Comments in <!-- --> are to be ignored
    -- All other terms are page names.
    datapage = mw.ustring.gsub(datapage, "<!%-%-.-%-%-%>", "") -- remove all comments
    datapage = datapage .. "\n== ==\n" -- add a final blank heading so it is easier to excise all headinged sections
    datapage = mw.ustring.gsub(datapage, "==([^\n]-)==", "[end content token][begin heading token]%1[end heading token]")
    getsection = mw.ustring.gmatch(datapage, "%[begin heading token%](.-)%[end heading token%](.-)%[end content token%]")
    local outtable = {} -- sequence
    local entries = {} -- sequence
    local rating = {} -- dictionary
    local category = {} -- dictionary
    section = ""
    while section do
    	-- within each section, there are commands in [ ] which affect the content that follows, sometimes using the current heading.
    	-- ONLY specific commands in [ ] listed below are parsed; the rest don't count and will end up being interpreted as pages.
        section = mw.ustring.gsub(section, "%[%s*autoindex%s*%]", "[end data token][begin command token]autoindex[end command token]")
    	section = mw.ustring.gsub(section, "%[%s*index%s*%]", "[end data token][begin command token]index[end command token]")
    	section = mw.ustring.gsub(section, "%[%s*rating%s*%]", "[end data token][begin command token]rating[end command token]")
    	section = mw.ustring.gsub(section, "%[%s*keyword %-%s*%]", "[end data token][begin command token]keyword -[end command token]")
    	section = mw.ustring.gsub(section, "%[%s*category%s*%]", "[end data token][begin command token]category[end command token]")	
    	section = section .. "\n[end data token]" -- add a final blank command to make it easier to excise all commands
    	getcommand = mw.ustring.gmatch(section, "%[begin command token%](.-)%[end command token%](.-)%[end data token%]")
    	command = " "
    	while command do
    		command, data = getcommand()
    		data = cleanup(data or "")
    		if "autoindex" == command then
    			command = "index"
    			data = p.getAll(frame)
    		end
    		if "index" == command then
    		   	local getentry = mw.ustring.gmatch(data, "([^\n\t]+)")
    		    local entry
    		    while true do
    		    	entry = getentry()
    		    	if (not entry) then break end
    		    	table.insert(entries, cleanup(entry or ""))
    		    end
    		elseif "keyword -" == command then
                        data = mw.text.trim(data)
    			if data ~= "" then				
    			    local i
    			    for i = #entries,1,-1 do -- reverse order so the numbers stay accurate!
    			        if mw.ustring.match(entries[i],data) then
    			        	table.remove(entries,i)
    			        end
    			    end
    			end
        	elseif "category" == command then
    		    dictinsert(category, data, heading)
			elseif "rating" == command then
				dictinsert(rating, data, heading)
			end
        end
        heading, section = getsection()
        heading = cleanup(heading or "")
    end
    table.insert(outtable, '\n{| class="wikitable sortable"')
    table.insert(outtable, '\n!Module name')
    table.insert(outtable, '\n!Category')
    table.insert(outtable, '\n!Rating')
    for i = 1, #entries do
    	table.insert(outtable, "\n|-")
    	local entry = entries[i] or ""  -- better not be nil...
    	table.insert(outtable, "\n|")
    	table.insert(outtable, "[[Module:" .. entry .. "|" .. entry .. "]] ([[Module talk:" .. entry .."|talk]])")
    	table.insert(outtable, "\n|")
    	table.insert(outtable, category[entry] or "")
    	table.insert(outtable, "\n|")
    	table.insert(outtable, rating[entry] or "")
    end
    table.insert(outtable, "\n|}") -- end wikitable
    local outdata = table.concat(outtable)
	return outdata..debuglog
end

return p