From 35a201cc8ef0c3f5b2df88d2e528aabee1048348 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Fri, 30 Apr 2021 18:47:09 +0200 Subject: Initial/Final commit --- libxml2-2.9.10/doc/index.py | 1258 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1258 insertions(+) create mode 100755 libxml2-2.9.10/doc/index.py (limited to 'libxml2-2.9.10/doc/index.py') diff --git a/libxml2-2.9.10/doc/index.py b/libxml2-2.9.10/doc/index.py new file mode 100755 index 0000000..77f9692 --- /dev/null +++ b/libxml2-2.9.10/doc/index.py @@ -0,0 +1,1258 @@ +#!/usr/bin/python -u +# +# imports the API description and fills up a database with +# name relevance to modules, functions or web pages +# +# Operation needed: +# ================= +# +# install mysqld, the python wrappers for mysql and libxml2, start mysqld +# Change the root passwd of mysql: +# mysqladmin -u root password new_password +# Create the new database xmlsoft +# mysqladmin -p create xmlsoft +# Create a database user 'veillard' and give him passord access +# change veillard and abcde with the right user name and passwd +# mysql -p +# password: +# mysql> GRANT ALL PRIVILEGES ON xmlsoft TO veillard@localhost +# IDENTIFIED BY 'abcde' WITH GRANT OPTION; +# +# As the user check the access: +# mysql -p xmlsoft +# Enter password: +# Welcome to the MySQL monitor.... +# mysql> use xmlsoft +# Database changed +# mysql> quit +# Bye +# +# Then run the script in the doc subdir, it will create the symbols and +# word tables and populate them with informations extracted from +# the libxml2-api.xml API description, and make them accessible read-only +# by nobody@loaclhost the user expected to be Apache's one +# +# On the Apache configuration, make sure you have php support enabled +# + +import MySQLdb +import libxml2 +import sys +import string +import os + +# +# We are not interested in parsing errors here +# +def callback(ctx, str): + return +libxml2.registerErrorHandler(callback, None) + +# +# The dictionary of tables required and the SQL command needed +# to create them +# +TABLES={ + "symbols" : """CREATE TABLE symbols ( + name varchar(255) BINARY NOT NULL, + module varchar(255) BINARY NOT NULL, + type varchar(25) NOT NULL, + descr varchar(255), + UNIQUE KEY name (name), + KEY module (module))""", + "words" : """CREATE TABLE words ( + name varchar(50) BINARY NOT NULL, + symbol varchar(255) BINARY NOT NULL, + relevance int, + KEY name (name), + KEY symbol (symbol), + UNIQUE KEY ID (name, symbol))""", + "wordsHTML" : """CREATE TABLE wordsHTML ( + name varchar(50) BINARY NOT NULL, + resource varchar(255) BINARY NOT NULL, + section varchar(255), + id varchar(50), + relevance int, + KEY name (name), + KEY resource (resource), + UNIQUE KEY ref (name, resource))""", + "wordsArchive" : """CREATE TABLE wordsArchive ( + name varchar(50) BINARY NOT NULL, + ID int(11) NOT NULL, + relevance int, + KEY name (name), + UNIQUE KEY ref (name, ID))""", + "pages" : """CREATE TABLE pages ( + resource varchar(255) BINARY NOT NULL, + title varchar(255) BINARY NOT NULL, + UNIQUE KEY name (resource))""", + "archives" : """CREATE TABLE archives ( + ID int(11) NOT NULL auto_increment, + resource varchar(255) BINARY NOT NULL, + title varchar(255) BINARY NOT NULL, + UNIQUE KEY id (ID,resource(255)), + INDEX (ID), + INDEX (resource))""", + "Queries" : """CREATE TABLE Queries ( + ID int(11) NOT NULL auto_increment, + Value varchar(50) NOT NULL, + Count int(11) NOT NULL, + UNIQUE KEY id (ID,Value(35)), + INDEX (ID))""", + "AllQueries" : """CREATE TABLE AllQueries ( + ID int(11) NOT NULL auto_increment, + Value varchar(50) NOT NULL, + Count int(11) NOT NULL, + UNIQUE KEY id (ID,Value(35)), + INDEX (ID))""", +} + +# +# The XML API description file to parse +# +API="libxml2-api.xml" +DB=None + +######################################################################### +# # +# MySQL database interfaces # +# # +######################################################################### +def createTable(db, name): + global TABLES + + if db == None: + return -1 + if name == None: + return -1 + c = db.cursor() + + ret = c.execute("DROP TABLE IF EXISTS %s" % (name)) + if ret == 1: + print "Removed table %s" % (name) + print "Creating table %s" % (name) + try: + ret = c.execute(TABLES[name]) + except: + print "Failed to create table %s" % (name) + return -1 + return ret + +def checkTables(db, verbose = 1): + global TABLES + + if db == None: + return -1 + c = db.cursor() + nbtables = c.execute("show tables") + if verbose: + print "Found %d tables" % (nbtables) + tables = {} + i = 0 + while i < nbtables: + l = c.fetchone() + name = l[0] + tables[name] = {} + i = i + 1 + + for table in TABLES.keys(): + if not tables.has_key(table): + print "table %s missing" % (table) + createTable(db, table) + try: + ret = c.execute("SELECT count(*) from %s" % table); + row = c.fetchone() + if verbose: + print "Table %s contains %d records" % (table, row[0]) + except: + print "Troubles with table %s : repairing" % (table) + ret = c.execute("repair table %s" % table); + print "repairing returned %d" % (ret) + ret = c.execute("SELECT count(*) from %s" % table); + row = c.fetchone() + print "Table %s contains %d records" % (table, row[0]) + if verbose: + print "checkTables finished" + + # make sure apache can access the tables read-only + try: + ret = c.execute("GRANT SELECT ON xmlsoft.* TO nobody@localhost") + ret = c.execute("GRANT INSERT,SELECT,UPDATE ON xmlsoft.Queries TO nobody@localhost") + except: + pass + return 0 + +def openMySQL(db="xmlsoft", passwd=None, verbose = 1): + global DB + + if passwd == None: + try: + passwd = os.environ["MySQL_PASS"] + except: + print "No password available, set environment MySQL_PASS" + sys.exit(1) + + DB = MySQLdb.connect(passwd=passwd, db=db) + if DB == None: + return -1 + ret = checkTables(DB, verbose) + return ret + +def updateWord(name, symbol, relevance): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if name == None: + return -1 + if symbol == None: + return -1 + + c = DB.cursor() + try: + ret = c.execute( +"""INSERT INTO words (name, symbol, relevance) VALUES ('%s','%s', %d)""" % + (name, symbol, relevance)) + except: + try: + ret = c.execute( + """UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'""" % + (relevance, name, symbol)) + except: + print "Update word (%s, %s, %s) failed command" % (name, symbol, relevance) + print "UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'" % (relevance, name, symbol) + print sys.exc_type, sys.exc_value + return -1 + + return ret + +def updateSymbol(name, module, type, desc): + global DB + + updateWord(name, name, 50) + if DB == None: + openMySQL() + if DB == None: + return -1 + if name == None: + return -1 + if module == None: + return -1 + if type == None: + return -1 + + try: + desc = string.replace(desc, "'", " ") + l = string.split(desc, ".") + desc = l[0] + desc = desc[0:99] + except: + desc = "" + + c = DB.cursor() + try: + ret = c.execute( +"""INSERT INTO symbols (name, module, type, descr) VALUES ('%s','%s', '%s', '%s')""" % + (name, module, type, desc)) + except: + try: + ret = c.execute( +"""UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % + (module, type, desc, name)) + except: + print "Update symbol (%s, %s, %s) failed command" % (name, module, type) + print """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % (module, type, desc, name) + print sys.exc_type, sys.exc_value + return -1 + + return ret + +def addFunction(name, module, desc = ""): + return updateSymbol(name, module, 'function', desc) + +def addMacro(name, module, desc = ""): + return updateSymbol(name, module, 'macro', desc) + +def addEnum(name, module, desc = ""): + return updateSymbol(name, module, 'enum', desc) + +def addStruct(name, module, desc = ""): + return updateSymbol(name, module, 'struct', desc) + +def addConst(name, module, desc = ""): + return updateSymbol(name, module, 'const', desc) + +def addType(name, module, desc = ""): + return updateSymbol(name, module, 'type', desc) + +def addFunctype(name, module, desc = ""): + return updateSymbol(name, module, 'functype', desc) + +def addPage(resource, title): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if resource == None: + return -1 + + c = DB.cursor() + try: + ret = c.execute( + """INSERT INTO pages (resource, title) VALUES ('%s','%s')""" % + (resource, title)) + except: + try: + ret = c.execute( + """UPDATE pages SET title='%s' WHERE resource='%s'""" % + (title, resource)) + except: + print "Update symbol (%s, %s, %s) failed command" % (name, module, type) + print """UPDATE pages SET title='%s' WHERE resource='%s'""" % (title, resource) + print sys.exc_type, sys.exc_value + return -1 + + return ret + +def updateWordHTML(name, resource, desc, id, relevance): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if name == None: + return -1 + if resource == None: + return -1 + if id == None: + id = "" + if desc == None: + desc = "" + else: + try: + desc = string.replace(desc, "'", " ") + desc = desc[0:99] + except: + desc = "" + + c = DB.cursor() + try: + ret = c.execute( +"""INSERT INTO wordsHTML (name, resource, section, id, relevance) VALUES ('%s','%s', '%s', '%s', '%d')""" % + (name, resource, desc, id, relevance)) + except: + try: + ret = c.execute( +"""UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % + (desc, id, relevance, name, resource)) + except: + print "Update symbol (%s, %s, %d) failed command" % (name, resource, relevance) + print """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % (desc, id, relevance, name, resource) + print sys.exc_type, sys.exc_value + return -1 + + return ret + +def checkXMLMsgArchive(url): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if url == None: + return -1 + + c = DB.cursor() + try: + ret = c.execute( + """SELECT ID FROM archives WHERE resource='%s'""" % (url)) + row = c.fetchone() + if row == None: + return -1 + except: + return -1 + + return row[0] + +def addXMLMsgArchive(url, title): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if url == None: + return -1 + if title == None: + title = "" + else: + title = string.replace(title, "'", " ") + title = title[0:99] + + c = DB.cursor() + try: + cmd = """INSERT INTO archives (resource, title) VALUES ('%s','%s')""" % (url, title) + ret = c.execute(cmd) + cmd = """SELECT ID FROM archives WHERE resource='%s'""" % (url) + ret = c.execute(cmd) + row = c.fetchone() + if row == None: + print "addXMLMsgArchive failed to get the ID: %s" % (url) + return -1 + except: + print "addXMLMsgArchive failed command: %s" % (cmd) + return -1 + + return((int)(row[0])) + +def updateWordArchive(name, id, relevance): + global DB + + if DB == None: + openMySQL() + if DB == None: + return -1 + if name == None: + return -1 + if id == None: + return -1 + + c = DB.cursor() + try: + ret = c.execute( +"""INSERT INTO wordsArchive (name, id, relevance) VALUES ('%s', '%d', '%d')""" % + (name, id, relevance)) + except: + try: + ret = c.execute( +"""UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" % + (relevance, name, id)) + except: + print "Update word archive (%s, %d, %d) failed command" % (name, id, relevance) + print """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" % (relevance, name, id) + print sys.exc_type, sys.exc_value + return -1 + + return ret + +######################################################################### +# # +# Word dictionary and analysis routines # +# # +######################################################################### + +# +# top 100 english word without the one len < 3 + own set +# +dropWords = { + 'the':0, 'this':0, 'can':0, 'man':0, 'had':0, 'him':0, 'only':0, + 'and':0, 'not':0, 'been':0, 'other':0, 'even':0, 'are':0, 'was':0, + 'new':0, 'most':0, 'but':0, 'when':0, 'some':0, 'made':0, 'from':0, + 'who':0, 'could':0, 'after':0, 'that':0, 'will':0, 'time':0, 'also':0, + 'have':0, 'more':0, 'these':0, 'did':0, 'was':0, 'two':0, 'many':0, + 'they':0, 'may':0, 'before':0, 'for':0, 'which':0, 'out':0, 'then':0, + 'must':0, 'one':0, 'through':0, 'with':0, 'you':0, 'said':0, + 'first':0, 'back':0, 'were':0, 'what':0, 'any':0, 'years':0, 'his':0, + 'her':0, 'where':0, 'all':0, 'its':0, 'now':0, 'much':0, 'she':0, + 'about':0, 'such':0, 'your':0, 'there':0, 'into':0, 'like':0, 'may':0, + 'would':0, 'than':0, 'our':0, 'well':0, 'their':0, 'them':0, 'over':0, + 'down':0, + 'net':0, 'www':0, 'bad':0, 'Okay':0, 'bin':0, 'cur':0, +} + +wordsDict = {} +wordsDictHTML = {} +wordsDictArchive = {} + +def cleanupWordsString(str): + str = string.replace(str, ".", " ") + str = string.replace(str, "!", " ") + str = string.replace(str, "?", " ") + str = string.replace(str, ",", " ") + str = string.replace(str, "'", " ") + str = string.replace(str, '"', " ") + str = string.replace(str, ";", " ") + str = string.replace(str, "(", " ") + str = string.replace(str, ")", " ") + str = string.replace(str, "{", " ") + str = string.replace(str, "}", " ") + str = string.replace(str, "<", " ") + str = string.replace(str, ">", " ") + str = string.replace(str, "=", " ") + str = string.replace(str, "/", " ") + str = string.replace(str, "*", " ") + str = string.replace(str, ":", " ") + str = string.replace(str, "#", " ") + str = string.replace(str, "\\", " ") + str = string.replace(str, "\n", " ") + str = string.replace(str, "\r", " ") + str = string.replace(str, "\xc2", " ") + str = string.replace(str, "\xa0", " ") + return str + +def cleanupDescrString(str): + str = string.replace(str, "'", " ") + str = string.replace(str, "\n", " ") + str = string.replace(str, "\r", " ") + str = string.replace(str, "\xc2", " ") + str = string.replace(str, "\xa0", " ") + l = string.split(str) + str = string.join(str) + return str + +def splitIdentifier(str): + ret = [] + while str != "": + cur = string.lower(str[0]) + str = str[1:] + if ((cur < 'a') or (cur > 'z')): + continue + while (str != "") and (str[0] >= 'A') and (str[0] <= 'Z'): + cur = cur + string.lower(str[0]) + str = str[1:] + while (str != "") and (str[0] >= 'a') and (str[0] <= 'z'): + cur = cur + str[0] + str = str[1:] + while (str != "") and (str[0] >= '0') and (str[0] <= '9'): + str = str[1:] + ret.append(cur) + return ret + +def addWord(word, module, symbol, relevance): + global wordsDict + + if word == None or len(word) < 3: + return -1 + if module == None or symbol == None: + return -1 + if dropWords.has_key(word): + return 0 + if ord(word[0]) > 0x80: + return 0 + + if wordsDict.has_key(word): + d = wordsDict[word] + if d == None: + return 0 + if len(d) > 500: + wordsDict[word] = None + return 0 + try: + relevance = relevance + d[(module, symbol)] + except: + pass + else: + wordsDict[word] = {} + wordsDict[word][(module, symbol)] = relevance + return relevance + +def addString(str, module, symbol, relevance): + if str == None or len(str) < 3: + return -1 + ret = 0 + str = cleanupWordsString(str) + l = string.split(str) + for word in l: + if len(word) > 2: + ret = ret + addWord(word, module, symbol, 5) + + return ret + +def addWordHTML(word, resource, id, section, relevance): + global wordsDictHTML + + if word == None or len(word) < 3: + return -1 + if resource == None or section == None: + return -1 + if dropWords.has_key(word): + return 0 + if ord(word[0]) > 0x80: + return 0 + + section = cleanupDescrString(section) + + if wordsDictHTML.has_key(word): + d = wordsDictHTML[word] + if d == None: + print "skipped %s" % (word) + return 0 + try: + (r,i,s) = d[resource] + if i != None: + id = i + if s != None: + section = s + relevance = relevance + r + except: + pass + else: + wordsDictHTML[word] = {} + d = wordsDictHTML[word]; + d[resource] = (relevance, id, section) + return relevance + +def addStringHTML(str, resource, id, section, relevance): + if str == None or len(str) < 3: + return -1 + ret = 0 + str = cleanupWordsString(str) + l = string.split(str) + for word in l: + if len(word) > 2: + try: + r = addWordHTML(word, resource, id, section, relevance) + if r < 0: + print "addWordHTML failed: %s %s" % (word, resource) + ret = ret + r + except: + print "addWordHTML failed: %s %s %d" % (word, resource, relevance) + print sys.exc_type, sys.exc_value + + return ret + +def addWordArchive(word, id, relevance): + global wordsDictArchive + + if word == None or len(word) < 3: + return -1 + if id == None or id == -1: + return -1 + if dropWords.has_key(word): + return 0 + if ord(word[0]) > 0x80: + return 0 + + if wordsDictArchive.has_key(word): + d = wordsDictArchive[word] + if d == None: + print "skipped %s" % (word) + return 0 + try: + r = d[id] + relevance = relevance + r + except: + pass + else: + wordsDictArchive[word] = {} + d = wordsDictArchive[word]; + d[id] = relevance + return relevance + +def addStringArchive(str, id, relevance): + if str == None or len(str) < 3: + return -1 + ret = 0 + str = cleanupWordsString(str) + l = string.split(str) + for word in l: + i = len(word) + if i > 2: + try: + r = addWordArchive(word, id, relevance) + if r < 0: + print "addWordArchive failed: %s %s" % (word, id) + else: + ret = ret + r + except: + print "addWordArchive failed: %s %s %d" % (word, id, relevance) + print sys.exc_type, sys.exc_value + return ret + +######################################################################### +# # +# XML API description analysis # +# # +######################################################################### + +def loadAPI(filename): + doc = libxml2.parseFile(filename) + print "loaded %s" % (filename) + return doc + +def foundExport(file, symbol): + if file == None: + return 0 + if symbol == None: + return 0 + addFunction(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + return 1 + +def analyzeAPIFile(top): + count = 0 + name = top.prop("name") + cur = top.children + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "exports": + count = count + foundExport(name, cur.prop("symbol")) + else: + print "unexpected element %s in API doc " % (name) + cur = cur.next + return count + +def analyzeAPIFiles(top): + count = 0 + cur = top.children + + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "file": + count = count + analyzeAPIFile(cur) + else: + print "unexpected element %s in API doc " % (cur.name) + cur = cur.next + return count + +def analyzeAPIEnum(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + addEnum(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + + return 1 + +def analyzeAPIConst(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + addConst(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + + return 1 + +def analyzeAPIType(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + addType(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + return 1 + +def analyzeAPIFunctype(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + addFunctype(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + return 1 + +def analyzeAPIStruct(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + addStruct(symbol, file) + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + + info = top.prop("info") + if info != None: + info = string.replace(info, "'", " ") + info = string.strip(info) + l = string.split(info) + for word in l: + if len(word) > 2: + addWord(word, file, symbol, 5) + return 1 + +def analyzeAPIMacro(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + symbol = string.replace(symbol, "'", " ") + symbol = string.strip(symbol) + + info = None + cur = top.children + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "info": + info = cur.content + break + cur = cur.next + + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + + if info == None: + addMacro(symbol, file) + print "Macro %s description has no " % (symbol) + return 0 + + info = string.replace(info, "'", " ") + info = string.strip(info) + addMacro(symbol, file, info) + l = string.split(info) + for word in l: + if len(word) > 2: + addWord(word, file, symbol, 5) + return 1 + +def analyzeAPIFunction(top): + file = top.prop("file") + if file == None: + return 0 + symbol = top.prop("name") + if symbol == None: + return 0 + + symbol = string.replace(symbol, "'", " ") + symbol = string.strip(symbol) + info = None + cur = top.children + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "info": + info = cur.content + elif cur.name == "return": + rinfo = cur.prop("info") + if rinfo != None: + rinfo = string.replace(rinfo, "'", " ") + rinfo = string.strip(rinfo) + addString(rinfo, file, symbol, 7) + elif cur.name == "arg": + ainfo = cur.prop("info") + if ainfo != None: + ainfo = string.replace(ainfo, "'", " ") + ainfo = string.strip(ainfo) + addString(ainfo, file, symbol, 5) + name = cur.prop("name") + if name != None: + name = string.replace(name, "'", " ") + name = string.strip(name) + addWord(name, file, symbol, 7) + cur = cur.next + if info == None: + print "Function %s description has no " % (symbol) + addFunction(symbol, file, "") + else: + info = string.replace(info, "'", " ") + info = string.strip(info) + addFunction(symbol, file, info) + addString(info, file, symbol, 5) + + l = splitIdentifier(symbol) + for word in l: + addWord(word, file, symbol, 10) + + return 1 + +def analyzeAPISymbols(top): + count = 0 + cur = top.children + + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "macro": + count = count + analyzeAPIMacro(cur) + elif cur.name == "function": + count = count + analyzeAPIFunction(cur) + elif cur.name == "const": + count = count + analyzeAPIConst(cur) + elif cur.name == "typedef": + count = count + analyzeAPIType(cur) + elif cur.name == "struct": + count = count + analyzeAPIStruct(cur) + elif cur.name == "enum": + count = count + analyzeAPIEnum(cur) + elif cur.name == "functype": + count = count + analyzeAPIFunctype(cur) + else: + print "unexpected element %s in API doc " % (cur.name) + cur = cur.next + return count + +def analyzeAPI(doc): + count = 0 + if doc == None: + return -1 + root = doc.getRootElement() + if root.name != "api": + print "Unexpected root name" + return -1 + cur = root.children + while cur != None: + if cur.type == 'text': + cur = cur.next + continue + if cur.name == "files": + pass +# count = count + analyzeAPIFiles(cur) + elif cur.name == "symbols": + count = count + analyzeAPISymbols(cur) + else: + print "unexpected element %s in API doc" % (cur.name) + cur = cur.next + return count + +######################################################################### +# # +# Web pages parsing and analysis # +# # +######################################################################### + +import glob + +def analyzeHTMLText(doc, resource, p, section, id): + words = 0 + try: + content = p.content + words = words + addStringHTML(content, resource, id, section, 5) + except: + return -1 + return words + +def analyzeHTMLPara(doc, resource, p, section, id): + words = 0 + try: + content = p.content + words = words + addStringHTML(content, resource, id, section, 5) + except: + return -1 + return words + +def analyzeHTMLPre(doc, resource, p, section, id): + words = 0 + try: + content = p.content + words = words + addStringHTML(content, resource, id, section, 5) + except: + return -1 + return words + +def analyzeHTML(doc, resource, p, section, id): + words = 0 + try: + content = p.content + words = words + addStringHTML(content, resource, id, section, 5) + except: + return -1 + return words + +def analyzeHTML(doc, resource): + para = 0; + ctxt = doc.xpathNewContext() + try: + res = ctxt.xpathEval("//head/title") + title = res[0].content + except: + title = "Page %s" % (resource) + addPage(resource, title) + try: + items = ctxt.xpathEval("//h1 | //h2 | //h3 | //text()") + section = title + id = "" + for item in items: + if item.name == 'h1' or item.name == 'h2' or item.name == 'h3': + section = item.content + if item.prop("id"): + id = item.prop("id") + elif item.prop("name"): + id = item.prop("name") + elif item.type == 'text': + analyzeHTMLText(doc, resource, item, section, id) + para = para + 1 + elif item.name == 'p': + analyzeHTMLPara(doc, resource, item, section, id) + para = para + 1 + elif item.name == 'pre': + analyzeHTMLPre(doc, resource, item, section, id) + para = para + 1 + else: + print "Page %s, unexpected %s element" % (resource, item.name) + except: + print "Page %s: problem analyzing" % (resource) + print sys.exc_type, sys.exc_value + + return para + +def analyzeHTMLPages(): + ret = 0 + HTMLfiles = glob.glob("*.html") + glob.glob("tutorial/*.html") + for html in HTMLfiles: + if html[0:3] == "API": + continue + if html == "xml.html": + continue + try: + doc = libxml2.parseFile(html) + except: + doc = libxml2.htmlParseFile(html, None) + try: + res = analyzeHTML(doc, html) + print "Parsed %s : %d paragraphs" % (html, res) + ret = ret + 1 + except: + print "could not parse %s" % (html) + return ret + +######################################################################### +# # +# Mail archives parsing and analysis # +# # +######################################################################### + +import time + +def getXMLDateArchive(t = None): + if t == None: + t = time.time() + T = time.gmtime(t) + month = time.strftime("%B", T) + year = T[0] + url = "http://mail.gnome.org/archives/xml/%d-%s/date.html" % (year, month) + return url + +def scanXMLMsgArchive(url, title, force = 0): + if url == None or title == None: + return 0 + + ID = checkXMLMsgArchive(url) + if force == 0 and ID != -1: + return 0 + + if ID == -1: + ID = addXMLMsgArchive(url, title) + if ID == -1: + return 0 + + try: + print "Loading %s" % (url) + doc = libxml2.htmlParseFile(url, None); + except: + doc = None + if doc == None: + print "Failed to parse %s" % (url) + return 0 + + addStringArchive(title, ID, 20) + ctxt = doc.xpathNewContext() + texts = ctxt.xpathEval("//pre//text()") + for text in texts: + addStringArchive(text.content, ID, 5) + + return 1 + +def scanXMLDateArchive(t = None, force = 0): + global wordsDictArchive + + wordsDictArchive = {} + + url = getXMLDateArchive(t) + print "loading %s" % (url) + try: + doc = libxml2.htmlParseFile(url, None); + except: + doc = None + if doc == None: + print "Failed to parse %s" % (url) + return -1 + ctxt = doc.xpathNewContext() + anchors = ctxt.xpathEval("//a[@href]") + links = 0 + newmsg = 0 + for anchor in anchors: + href = anchor.prop("href") + if href == None or href[0:3] != "msg": + continue + try: + links = links + 1 + + msg = libxml2.buildURI(href, url) + title = anchor.content + if title != None and title[0:4] == 'Re: ': + title = title[4:] + if title != None and title[0:6] == '[xml] ': + title = title[6:] + newmsg = newmsg + scanXMLMsgArchive(msg, title, force) + + except: + pass + + return newmsg + + +######################################################################### +# # +# Main code: open the DB, the API XML and analyze it # +# # +######################################################################### +def analyzeArchives(t = None, force = 0): + global wordsDictArchive + + ret = scanXMLDateArchive(t, force) + print "Indexed %d words in %d archive pages" % (len(wordsDictArchive), ret) + + i = 0 + skipped = 0 + for word in wordsDictArchive.keys(): + refs = wordsDictArchive[word] + if refs == None: + skipped = skipped + 1 + continue; + for id in refs.keys(): + relevance = refs[id] + updateWordArchive(word, id, relevance) + i = i + 1 + + print "Found %d associations in HTML pages" % (i) + +def analyzeHTMLTop(): + global wordsDictHTML + + ret = analyzeHTMLPages() + print "Indexed %d words in %d HTML pages" % (len(wordsDictHTML), ret) + + i = 0 + skipped = 0 + for word in wordsDictHTML.keys(): + refs = wordsDictHTML[word] + if refs == None: + skipped = skipped + 1 + continue; + for resource in refs.keys(): + (relevance, id, section) = refs[resource] + updateWordHTML(word, resource, section, id, relevance) + i = i + 1 + + print "Found %d associations in HTML pages" % (i) + +def analyzeAPITop(): + global wordsDict + global API + + try: + doc = loadAPI(API) + ret = analyzeAPI(doc) + print "Analyzed %d blocs" % (ret) + doc.freeDoc() + except: + print "Failed to parse and analyze %s" % (API) + print sys.exc_type, sys.exc_value + sys.exit(1) + + print "Indexed %d words" % (len(wordsDict)) + i = 0 + skipped = 0 + for word in wordsDict.keys(): + refs = wordsDict[word] + if refs == None: + skipped = skipped + 1 + continue; + for (module, symbol) in refs.keys(): + updateWord(word, symbol, refs[(module, symbol)]) + i = i + 1 + + print "Found %d associations, skipped %d words" % (i, skipped) + +def usage(): + print "Usage index.py [--force] [--archive] [--archive-year year] [--archive-month month] [--API] [--docs]" + sys.exit(1) + +def main(): + try: + openMySQL() + except: + print "Failed to open the database" + print sys.exc_type, sys.exc_value + sys.exit(1) + + args = sys.argv[1:] + force = 0 + if args: + i = 0 + while i < len(args): + if args[i] == '--force': + force = 1 + elif args[i] == '--archive': + analyzeArchives(None, force) + elif args[i] == '--archive-year': + i = i + 1; + year = args[i] + months = ["January" , "February", "March", "April", "May", + "June", "July", "August", "September", "October", + "November", "December"]; + for month in months: + try: + str = "%s-%s" % (year, month) + T = time.strptime(str, "%Y-%B") + t = time.mktime(T) + 3600 * 24 * 10; + analyzeArchives(t, force) + except: + print "Failed to index month archive:" + print sys.exc_type, sys.exc_value + elif args[i] == '--archive-month': + i = i + 1; + month = args[i] + try: + T = time.strptime(month, "%Y-%B") + t = time.mktime(T) + 3600 * 24 * 10; + analyzeArchives(t, force) + except: + print "Failed to index month archive:" + print sys.exc_type, sys.exc_value + elif args[i] == '--API': + analyzeAPITop() + elif args[i] == '--docs': + analyzeHTMLTop() + else: + usage() + i = i + 1 + else: + usage() + +if __name__ == "__main__": + main() -- cgit v1.2.3