#!/usr/bin/env python

doc = """
mean             will asume ./
mean ... in ./doc   will take ./doc as root
mean ... in file    will take file a lonely file

{meta one complete sentence.... any thing till}
they are used for almost everything, _var_ [ref] TIMELINE and QUERY
all information come from meta in all file. Meta can be declared many time, they just stackup.

_variable_ 
they are simply used and intended to be defined later
used for name of thing and character. you never know the namr of some before 
they are mannipulated by adding meta : {_Variable_ is ....}
you can list variable using mean var
and read one using mean var _variable_ wtf
   note : mean will always complain if some var are unbounded

[reference]
when you reference something
the teggen toppa guren lagann[ttgl] is throwing...
you are giving more information about what are you saying
but without cluging the semantic
you can define any reference using mean about ttgl is a galaxy sized mecha

TIMELINE
in any file you can define a timeline, the order of event
	{timeline morning noon afternoon evening}
then every file can have something like
	{during morning}
	{during afternoon}
	{before evening}
	{after noon}
a file can have many time event and many timeline
but beware, no collision will be detected and may result in duplication
   note : if no timeline is setted, all file will appear in read order user is warned
   note : if one file have two location, last one is discarted and user is warned
   note : having many timeline can result in impreditible behavior, 
          hiearchy is based on dir order
   note : must consider every timelines when defining timestamp

QUERY
the ability select document by metadata
-with saga twilight starwars		where meta(saga) == "twilight starwars"
-without chapiter 3			where meta(chapiter) != "3"
-have chapiter paragraph		with at least a defined meta(paragraph)
-miss character				where meta(character) is undefined
it wont handle complex query, it is not suposed to become a turing complete language
but it will cover all reqiirement

multiple output
direct json -json
text output -text
ref output -reference
stat output -stat
show timeline -timeline
list ref -ref
list var -var
list meta -meta

TODO:
better documentation
file support on meta {metaname file://file/uri}
web support on meta {metaname http://www.wikipedia.com/...}
"""

import re, os, sys

def _debug(s):
	print("debug : %s" % s)
	return s

#####################################file

def lstFile(srcDir):
	"""
	walk all directory recursively and return a list of file
	"""
	if os.path.isfile(srcDir):
		return [srcDir]
	return	[	dirpath+'/'+filename
                        for dirpath,lstDirnames,lstFilenames in os.walk(srcDir)
                        if not re.search('\\.[A-Za-z]+',dirpath)	#not hidden dir
                        if dirpath != '.'
			for filename in lstFilenames
			if not re.search('\\.[A-Za-z]+',filename)	#not hidden file
			if filename != __file__	]

def loadText(filepath):
	with open(filepath) as file:                
		content = file.read().replace('\r','\n')
		return content

def loadContent(lstPath):
	if not lstPath:
		lstPath = [['.']]
	content = [	(
				filename,
				loadText(filename)
			)
			for lstDir in lstPath
			for srcDir in lstDir
			for filename in lstFile(srcDir)	]
	return content

####################CONTENT

def findMeta(content):
	metaLst = []
	varLst = []
	refLst = []
	for filename, text in content:
		#meta
		for match in re.findall(r'\{[^\}]+\}',text):
			parse = match[1:len(match)-1].strip().split(' ')
			if parse:
				meta = parse[0]
				data = ' '.join(parse[1:])
				if re.match(r'https?//.+',data):
					print("got web url : %s \n but it is not supported yet" % data)
				if re.match(r'file://.+',data):
					print("got file path : %s \n but it is not supported yet" % data)
				metaLst.append((meta,data,filename))
	return metaLst

def getOrderedFileLst(metaLst):
	import itertools
	def order(timelines, events):
		if not timelines:
			return events
		ordered = []
		for timestamp in timelines[0]:
			before = []
			during = []
			after = []
			tmp = [	(event,lst,when,ts)
				for event,lst in events
				for when,ts,_ in lst
				if ts == timestamp	]
			during = [	(event,lst)
				 	for event,lst,when,ts in tmp
					if when == 'during'	]
                        after = [	(event,lst)
                                        for event,lst,when,ts in tmp
					if when == 'after'	]
                        before = [      (event,lst)
                                        for event,lst,when,ts in tmp
					if when == 'before'	]
			ordered += order(timelines[1:],before)
			ordered += order(timelines[1:],during)
			ordered += order(timelines[1:],after)
		lostEvent = [	event
				for event in events
				if event not in ordered	]
		if lostEvent:
			#lostFile = [filename for filename,_ in lostEvent]
			#print ('warning these file [%s] cant place thenself in timeline [%s], they are placed at the end of the event' % (' '.join(lostFile),' '.join(timelines[0])))
			ordered += order(timelines[1:],lostEvent)
		return ordered
	timelines = [	data.split(' ')
			for meta,data,filename in metaLst
			if meta == 'timeline'	]
	eventLst = [	(meta,data,filename)
			for meta,data,filename in metaLst
			if meta in ['during','after','before']	]
	events = [	(filename,list(evLst))
			for filename,evLst 
			in itertools.groupby(eventLst,lambda a : a[2])	]
	orderedLst = [ filename for filename,_ in order(timelines,events) ]
	#make sure no duplicate
	bag = set()
	orderedLst2 = []
	for filename in orderedLst:
		if filename in bag:
			print('warning the file %s refer two place in timeline' % filename)
		else:
			bag.add(filename)
			orderedLst2.append(filename)
	#check who is missing
	lostFileLst = set([	filename 
				for _,_,filename in metaLst 
				if filename not in orderedLst2	])
	if lostFileLst:
		print('warning , these(s) file(s) [%s] cant place itself in any timeline, they are place at the end' % ' '.join(lostFileLst))
		orderedLst2 += lostFileLst
	#finaly
	return orderedLst2

def sortContent(content,sortedFilenameLst):
	sortedContent = [	(filename,text)
				for filename in sortedFilenameLst
				for filen,text in content
				if filen==filename	]
	return sortedContent

def query(metaLst,select=[],reject=[],have=[],miss=[]):
	if not select and not have:
		fileLst = [     filename
                                for _,_,filename in metaLst     ]
	else:
		fileLst = []
	if select:
        	fileLst += [	filename
	                       	for meta,data,filename in metaLst
        	               	for selectMeta in select
				if selectMeta
	                       	if selectMeta[0] == meta
				if ' '.join(selectMeta[1:]) == data	]  #any intersection
	if have:
		fileLst += [	filename
				for meta,data,filename in metaLst
				for havemetaLst in have
				for havemeta in havemetaLst
				if havemeta == meta	]
	remLst = []
	remLst += [	filename
			for meta,data,filename in metaLst
		       	for rejectMeta in reject
			if rejectMeta
		       	if rejectMeta[0] == meta
			if ' '.join(rejectMeta[1:]) == data	]
	remLst += [	filename
			for meta,data,filename in metaLst
			for missMetaLst in miss
			for missMeta in missMetaLst
			if missMeta == meta	]
	return list(set(fileLst)-set(remLst))

def selectContent(content,fileLst):
	return [	(filename,text)
			for filename,text in content
			if filename in fileLst	]
##############################TEXT

def getVarLst(text):
	#variable
	varLstThisFile = {}
	for match in re.findall(r'_[^ ]+_ ',text):
		if match in varLstThisFile:
			varLstThisFile[match] += 1
		else:
			varLstThisFile[match] = 1
	return [(var.strip(),score) for var, score in varLstThisFile.items()]

def getReferenceLst(text):
	refLst = []
	#reference
	for match in re.findall(r'[^\.]*\[[^\]]+\][^\.]*\.',text):
		#print('found reference %s' % str(match))
		refMatch = re.search(r'\[[^\]]+\]',match)
		if refMatch:
			#print ('ref is %s' % refMatch.group(1))
			refLst.append((refMatch.group(0),match)) 
	return refLst
##################################report

def filterMeta(text):
	newText = re.sub(r'\{[^\}]+\}','',text)
	while('\n\n' in newText):
		newText = newText.replace('\n\n','')
	return newText

def applyVariable(text,metaLst):
	metaDict = dict([	(meta,data)
				for meta,data,_ in metaLst	])
	varLst = getVarLst(text)
	newText = text
	for var,score in varLst:
		if var in metaDict:
			newText = newText.replace(var,metaDict[var])
	return newText

def extractReference(text, metaLst):
	metaDict = dict([	(meta,data)
				for meta,data,_ in metaLst	])
	newText = ''
	refLst = getReferenceLst(text)
	for ref,_ in refLst:
		if ref in metaDict:
			newText += '%s %s\n' % (ref,metaDict[ref])
	return newText

def getWordLst(text):
	assert isinstance(text,str)
	wordLst = [     word.strip().lower()
                        for sentence in text.split('.')
                        for word in sentence.split(' ')
			if word.strip()	]
	return wordLst

def getUniqueWordLst(text):
	assert isinstance(text,str)
	uniqueWordLst = list(set(getWordLst(text)))
	return uniqueWordLst

def getWordScore(word, text):
        assert isinstance(text,str)
	wordScore = len([	same
				for same in getWordLst(text)
				if same==word	])
	return wordScore

def getWordStat(text):
        assert isinstance(text,str)
	wordScore = [	(
				word,
				getWordScore(word,text)
			)
			for word in getUniqueWordLst(text)	]
	return wordScore
			
################################option
def what():
        import argparse
        arg = argparse.ArgumentParser('')
        arg.add_argument('-in',
			 dest='path',
			 nargs='+',
			 help='file path to consider', 
			 action='append',
			 default=[])
	arg.add_argument('-as',
			 dest='saveto',
			 nargs=1,
			 help='the output file',
			 default=None)
	arg.add_argument('-tpl',
			 dest='tpl',
			 nargs=1,
			 default=None,
			 help='the template used')
	arg.add_argument('-with',
			 dest='select',
			 nargs='+',
			 action='append',
			 default=[],
			 help='where meta is ...')
	arg.add_argument('-have',
			 dest='have',
			 nargs='+',
			 action='append',
			 default=[],
			 help='who have meta ...')
	arg.add_argument('-without',
			 dest='reject',
			 nargs='+',
			 action='append',
			 default=[],
			 help='where meta is not ...')
	arg.add_argument('-miss',
			 dest='miss',
			 nargs='+',
                         action='append',
                         default=[],
                         help='who miss meta ...')
	arg.add_argument('--stat',help='show stat',action='store_true')
	arg.add_argument('--text',help='show text',action='store_true')
        arg.add_argument('--dictionary',help='show reference dictionary',action='store_true')
        arg.add_argument('--variable',help='show variable',action='store_true')
        arg.add_argument('--reference',help='show reference',action='store_true')
        arg.add_argument('--meta',help='show meta data',action='store_true')
	arg.add_argument('--json',help='show complete report in json',action='store_true')

        ctx = arg.parse_args()
        return ctx	
#################################MAIN

def buildReport(content,select,reject,have,miss):
	metaLst = findMeta(content)
	#select using meta
	keepFileLst = query(metaLst,select,reject,have,miss)
	content = selectContent(content,keepFileLst)
	metaLst = findMeta(content)
	#sort using timeline
	order = getOrderedFileLst(metaLst)
	if order:
		content = sortContent(content,order)
	else:
		print('warning no timeline or no file to match any event, all file is used')

	allTextLst = [	applyVariable(filterMeta(text),metaLst)
			for _,text in content	]
	allText = '\n\n'.join(allTextLst)
	for varname,_ in getVarLst(allText):
		print("warning, unbound variable named %s" % varname)
	for ref in getReferenceLst(allText):
		print("warning, undefined reference %s in %s" % ref)
	allRef = extractReference(allText,metaLst)
	fileCnt = len(content)
	wordCnt = len(getWordLst(allText))
	wordAvg = wordCnt/fileCnt if fileCnt else 0
	uniqueCnt = len(getUniqueWordLst(allText))

	refLst = [	(ref,sentence,filename)
			for filename,text in content
			for ref,sentence in getReferenceLst(filterMeta(text))	]
	varLst = [	(var,score,filename)
			for filename,text in content
			for var,score in getVarLst(filterMeta(text))	]
	db = [	{
			'filename' : filename,
			'raw'	   : text,
			'text'	   : applyVariable(filterMeta(text),metaLst),
			'ref'      : extractReference(text,metaLst),
			'meta'	   : findMeta([(filename,text)])
		}
		for filename,text in content	]

	return {
		'text' : allText,
		'dict'  : allRef,
		'stat' : {
			'wordCnt' : wordCnt,
			'fileCnt' : fileCnt,
			'wordAvg' : wordAvg,
			'uniqueCnt' : uniqueCnt
                },
		'meta' : metaLst,
		'variable' : varLst,
		'reference' : refLst,
		'db' : db
	}

def applyTemplate(report,tpl):
	from string import Template
	template = Template(tpl)
	return template.substitute(report)

if __name__ == '__main__':
	statTpl = """
file count : $fileCnt
word count : $wordCnt
unique word count : $uniqueCnt
word per file : $wordAvg
"""
	ctx = what()
	content = loadContent(ctx.path)
	report = buildReport(content,
			     ctx.select,
			     ctx.reject,
			     ctx.have,
			     ctx.miss)

	text = ''
	if ctx.stat:
		text += applyTemplate(report['stat'],statTpl) + '\n'
	if ctx.text:
		text += report['text']+ '\n'
	if ctx.dictionary:
		text += report['dict'] + '\n'
	if ctx.meta:
		for meta in report['meta']:
			text += "%s =>\n %s\n from >>>%s\n" % meta
	if ctx.reference:
		for ref in report['reference']:
			text += "[%s] in\n \"%s\"\n from >>>%s\n" % ref
	if ctx.variable:
		for var in report['variable']:
			text += "%s %s times from >>>%s\n" % var
	if ctx.json:
		import json
		text += json.dumps(report)
	if ctx.saveto:
		with open(ctx.saveto[0],'w') as file:
			file.write(text)
			print('saved')
	else:
		print()
		print(text)