mirror of
https://github.com/RussellChamp/tilde-projects.git
synced 2026-10-28 00:22:34 +00:00
black
This commit is contained in:
38 files changed
+4549
-2523
No files matched your search
+87
-55
@@ -11,29 +11,29 @@ import operator
|
||||
MAX_NODES = 4
|
||||
|
||||
logfile = "/home/archangelic/irc/log"
|
||||
#logfile = "/home/jumblesale/Code/irc/log"
|
||||
# logfile = "/home/jumblesale/Code/irc/log"
|
||||
outfile = "/home/krowbar/logs/chatBesties.json"
|
||||
outCircle = "/home/krowbar/logs/chatcircle.json"
|
||||
timePeriod = calendar.timegm(time.gmtime()) - (2 * 7 * 24 * 60 * 60) #2 weeks
|
||||
timePeriod = calendar.timegm(time.gmtime()) - (2 * 7 * 24 * 60 * 60) # 2 weeks
|
||||
|
||||
userData = {} #hash keyed by "user" that contains a hash of mentioned other users with count
|
||||
userData = {} # hash keyed by "user" that contains a hash of mentioned other users with count
|
||||
nameFix = {
|
||||
'jumblesal': 'jumblesale',
|
||||
'hardmath1': 'kc',
|
||||
'hardmath123': 'kc',
|
||||
'bendorphan': 'endorphant',
|
||||
'endorphan': 'endorphant',
|
||||
'synergian': 'synergiance'
|
||||
}
|
||||
"jumblesal": "jumblesale",
|
||||
"hardmath1": "kc",
|
||||
"hardmath123": "kc",
|
||||
"bendorphan": "endorphant",
|
||||
"endorphan": "endorphant",
|
||||
"synergian": "synergiance",
|
||||
}
|
||||
|
||||
users = []
|
||||
#Get a list of all user names by checking the logs for people who have said things
|
||||
# Get a list of all user names by checking the logs for people who have said things
|
||||
with open(logfile, "r") as log:
|
||||
for line in log:
|
||||
try:
|
||||
time, user, message = line.split("\t", 3)
|
||||
if int(time) < timePeriod:
|
||||
continue #only add users who have spoken in the last period
|
||||
continue # only add users who have spoken in the last period
|
||||
if nameFix.has_key(user):
|
||||
user = nameFix[user]
|
||||
else:
|
||||
@@ -42,73 +42,105 @@ with open(logfile, "r") as log:
|
||||
if user not in users:
|
||||
users.append(user)
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
|
||||
d3data = {}
|
||||
d3data['nodes'] = []
|
||||
d3data["nodes"] = []
|
||||
|
||||
#re-read the log and this time look for instances of user names in messages
|
||||
# re-read the log and this time look for instances of user names in messages
|
||||
with open(logfile, "r") as log:
|
||||
for line in log:
|
||||
try:
|
||||
time, user, message = line.split("\t", 3)
|
||||
if int(time) < timePeriod:
|
||||
continue #only consider the past three weeks of chats
|
||||
continue # only consider the past three weeks of chats
|
||||
if nameFix.has_key(user):
|
||||
user = nameFix[user]
|
||||
else:
|
||||
user = user.lower()
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
for word in message.split(' '):
|
||||
word = re.sub('[^A-Za-z0-9]+', '', word)
|
||||
if word in users: #SOMEONE MENTIONED SOMEONE
|
||||
if userData.has_key(user): #This user is already set up
|
||||
if userData[user]['data'].has_key(word): #This user has mentioned this person before
|
||||
userData[user]['data'][word] += 1
|
||||
else: #This user never mentioned this person before
|
||||
userData[user]['data'][word] = 1
|
||||
#give both the target and mentioner a point
|
||||
else: #This user was never set up
|
||||
userData[user] = {} #make it a dictionary!
|
||||
userData[user]['data'] = {}
|
||||
userData[user]['data'][word] = 1
|
||||
userData[user]['score'] = 0
|
||||
userData[user]['id'] = len(d3data['nodes']) #so we know how to match people during the links phase
|
||||
d3data['nodes'].append({"name": user, "group": 1})
|
||||
if not userData.has_key(word): #check if the target has not been set up
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
for word in message.split(" "):
|
||||
word = re.sub("[^A-Za-z0-9]+", "", word)
|
||||
if word in users: # SOMEONE MENTIONED SOMEONE
|
||||
if userData.has_key(user): # This user is already set up
|
||||
if userData[user]["data"].has_key(
|
||||
word
|
||||
): # This user has mentioned this person before
|
||||
userData[user]["data"][word] += 1
|
||||
else: # This user never mentioned this person before
|
||||
userData[user]["data"][word] = 1
|
||||
# give both the target and mentioner a point
|
||||
else: # This user was never set up
|
||||
userData[user] = {} # make it a dictionary!
|
||||
userData[user]["data"] = {}
|
||||
userData[user]["data"][word] = 1
|
||||
userData[user]["score"] = 0
|
||||
userData[user]["id"] = len(
|
||||
d3data["nodes"]
|
||||
) # so we know how to match people during the links phase
|
||||
d3data["nodes"].append({"name": user, "group": 1})
|
||||
if not userData.has_key(
|
||||
word
|
||||
): # check if the target has not been set up
|
||||
userData[word] = {}
|
||||
userData[word]['data'] = {}
|
||||
userData[word]['score'] = 0
|
||||
userData[word]['id'] = len(d3data['nodes'])
|
||||
d3data['nodes'].append({"name": word, "group": 1})
|
||||
userData[user]['score'] += 1
|
||||
userData[word]['score'] += 1
|
||||
userData[word]["data"] = {}
|
||||
userData[word]["score"] = 0
|
||||
userData[word]["id"] = len(d3data["nodes"])
|
||||
d3data["nodes"].append({"name": word, "group": 1})
|
||||
userData[user]["score"] += 1
|
||||
userData[word]["score"] += 1
|
||||
|
||||
d3data['links'] = []
|
||||
#Now connect all the pople to their stuff
|
||||
d3data["links"] = []
|
||||
# Now connect all the pople to their stuff
|
||||
for user, values in userData.iteritems():
|
||||
#give the user a 'group' based on their total score
|
||||
d3data['nodes'][values['id']]['group'] = int(math.ceil(math.log(values['score'])))
|
||||
besties = sorted(values['data'].items(), key=operator.itemgetter(1), reverse=True)[0:MAX_NODES] #ONLY the top besties
|
||||
# give the user a 'group' based on their total score
|
||||
d3data["nodes"][values["id"]]["group"] = int(math.ceil(math.log(values["score"])))
|
||||
besties = sorted(values["data"].items(), key=operator.itemgetter(1), reverse=True)[
|
||||
0:MAX_NODES
|
||||
] # ONLY the top besties
|
||||
for target, score in besties:
|
||||
try:
|
||||
print "Adding link from " + user + " (" + str(values['id']) + ") to " + target + " (" + str(userData[target]['id']) + ") with strength " + str(score)
|
||||
d3data['links'].append({"source": values['id'], "target": userData[target]['id'], "value": math.ceil(math.sqrt(score))*2 })
|
||||
print(
|
||||
"Adding link from "
|
||||
+ user
|
||||
+ " ("
|
||||
+ str(values["id"])
|
||||
+ ") to "
|
||||
+ target
|
||||
+ " ("
|
||||
+ str(userData[target]["id"])
|
||||
+ ") with strength "
|
||||
+ str(score)
|
||||
)
|
||||
d3data["links"].append(
|
||||
{
|
||||
"source": values["id"],
|
||||
"target": userData[target]["id"],
|
||||
"value": math.ceil(math.sqrt(score)) * 2,
|
||||
}
|
||||
)
|
||||
except KeyError:
|
||||
print "! Error when trying to link " + user + " to " + target
|
||||
print("! Error when trying to link " + user + " to " + target)
|
||||
continue
|
||||
if len(values['data']) > MAX_NODES:
|
||||
print " ...ignoring " + str(len(values['data']) - MAX_NODES) + " more connections from " + user
|
||||
if len(values["data"]) > MAX_NODES:
|
||||
print(
|
||||
" ...ignoring "
|
||||
+ str(len(values["data"]) - MAX_NODES)
|
||||
+ " more connections from "
|
||||
+ user
|
||||
)
|
||||
|
||||
d3Circle = {}
|
||||
d3Circle['names'] = [''] * len(userData)
|
||||
d3Circle['matrix'] = [[0] * len(userData)] * len(userData)
|
||||
d3Circle["names"] = [""] * len(userData)
|
||||
d3Circle["matrix"] = [[0] * len(userData)] * len(userData)
|
||||
|
||||
for user, values in userData.iteritems():
|
||||
d3Circle['names'][values['id']] = user
|
||||
for name, score in values['data'].iteritems():
|
||||
d3Circle['matrix'][values['id']][userData[name]['id']] = score if score > 1 else 0
|
||||
d3Circle["names"][values["id"]] = user
|
||||
for name, score in values["data"].iteritems():
|
||||
d3Circle["matrix"][values["id"]][userData[name]["id"]] = (
|
||||
score if score > 1 else 0
|
||||
)
|
||||
|
||||
with open(outfile + ".tmp", "w") as tmpFile:
|
||||
tmpFile.write(json.dumps(d3data))
|
||||
|
||||
+11
-11
@@ -6,10 +6,10 @@ import calendar
|
||||
import shutil
|
||||
|
||||
logfile = "/home/archangelic/irc/log"
|
||||
#logfile = "/home/jumblesale/Code/irc/log"
|
||||
# logfile = "/home/jumblesale/Code/irc/log"
|
||||
outfile = "/home/krowbar/logs/userData.json"
|
||||
userData = {} #hash keyed by "user" that contains an array of timestamps
|
||||
#we only care about recent chats, let's say for the past couple weeks
|
||||
userData = {} # hash keyed by "user" that contains an array of timestamps
|
||||
# we only care about recent chats, let's say for the past couple weeks
|
||||
oneWeek = 7 * 24 * 60 * 60
|
||||
fiveMinutes = 5 * 60
|
||||
timeCutoff = calendar.timegm(time.gmtime()) - (5 * oneWeek)
|
||||
@@ -20,17 +20,17 @@ with open(logfile, "r") as log:
|
||||
time, user, message = line.split("\t", 3)
|
||||
time = int(time)
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
if time > timeCutoff:
|
||||
#standardize all times into a single "week"
|
||||
#time %= oneWeek
|
||||
#if the user already exists in the list
|
||||
# standardize all times into a single "week"
|
||||
# time %= oneWeek
|
||||
# if the user already exists in the list
|
||||
if user in userData:
|
||||
if time < userData[user][-1] + fiveMinutes:
|
||||
continue #only record times that are at least five minutes apart
|
||||
userData[user].append(time) #append the new time
|
||||
else: #if they are new
|
||||
userData[user] = [time] #create a new list
|
||||
continue # only record times that are at least five minutes apart
|
||||
userData[user].append(time) # append the new time
|
||||
else: # if they are new
|
||||
userData[user] = [time] # create a new list
|
||||
with open(outfile + ".tmp", "w") as tmpFile:
|
||||
tmpFile.write(json.dumps(userData))
|
||||
shutil.move(outfile + ".tmp", outfile)
|
||||
+29
-20
@@ -7,22 +7,27 @@ import re
|
||||
import shutil
|
||||
|
||||
logfile = "/home/archangelic/irc/log"
|
||||
#logfile = "/home/jumblesale/Code/irc/log"
|
||||
# logfile = "/home/jumblesale/Code/irc/log"
|
||||
outfile = "/home/krowbar/logs/chatcloud.json"
|
||||
#outfile = "/home/krowbar/logs/chatcloud_2016_10.json"
|
||||
# outfile = "/home/krowbar/logs/chatcloud_2016_10.json"
|
||||
|
||||
bannedUsersFile = "/home/krowbar/Code/python/bannedUsers"
|
||||
bannedWordsFile = "/home/krowbar/Code/python/bannedWords"
|
||||
wordData = {} # keyed by "word" that contains a count
|
||||
#we only care about recent chats, let's say for the past sixteen hours
|
||||
wordData = {} # keyed by "word" that contains a count
|
||||
# we only care about recent chats, let's say for the past sixteen hours
|
||||
timeTo = calendar.timegm(time.gmtime())
|
||||
#timeTo = calendar.timegm(time.strptime("1 Nov 16", "%d %b %y"))
|
||||
# timeTo = calendar.timegm(time.strptime("1 Nov 16", "%d %b %y"))
|
||||
|
||||
timeCutoff = timeTo - (16 * 60 * 60)
|
||||
#timeCutoff = calendar.timegm(time.strptime("1 Oct 16", "%d %b %y"))
|
||||
print "Generating word cloud based off words from " + str(timeCutoff) + " to " + str(timeTo)
|
||||
minOccurance = 3 #we'll have to reduce the minOccurances if we reduce the timeCutoff
|
||||
minLength = 3 #number of letters long
|
||||
# timeCutoff = calendar.timegm(time.strptime("1 Oct 16", "%d %b %y"))
|
||||
print(
|
||||
"Generating word cloud based off words from "
|
||||
+ str(timeCutoff)
|
||||
+ " to "
|
||||
+ str(timeTo)
|
||||
)
|
||||
minOccurance = 3 # we'll have to reduce the minOccurances if we reduce the timeCutoff
|
||||
minLength = 3 # number of letters long
|
||||
bannedWords = open(bannedWordsFile).read().splitlines()
|
||||
bannedUsers = open(bannedUsersFile).read().splitlines()
|
||||
|
||||
@@ -32,24 +37,28 @@ with open(logfile, "r") as log:
|
||||
time, user, message = line.split("\t", 3)
|
||||
time = int(time)
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
if user in bannedUsers:
|
||||
continue #We don't care what they say
|
||||
continue # We don't care what they say
|
||||
if time >= timeCutoff and time <= timeTo:
|
||||
#print "Processing line from " + user + " at " + str(time)
|
||||
for word in re.sub('[\'\"\`\/\\;:,.?!*&^\-()<>\{\}|_\[\]0-9]', ' ', message).lower().split():
|
||||
#changing symbols into spaces instead of stripping them avoids compounded words
|
||||
# print "Processing line from " + user + " at " + str(time)
|
||||
for word in (
|
||||
re.sub("['\"\`\/\\;:,.?!*&^\-()<>\{\}|_\[\]0-9]", " ", message)
|
||||
.lower()
|
||||
.split()
|
||||
):
|
||||
# changing symbols into spaces instead of stripping them avoids compounded words
|
||||
if len(word) < minLength or word in bannedWords:
|
||||
#print "Rejecting " + word
|
||||
# print "Rejecting " + word
|
||||
continue
|
||||
#if the word already exists in the list
|
||||
# if the word already exists in the list
|
||||
if word in wordData:
|
||||
wordData[word] += 1
|
||||
else: #if they are new
|
||||
else: # if they are new
|
||||
wordData[word] = 1
|
||||
#print "Added word: " + word
|
||||
wordData = {i:wordData[i] for i in wordData if wordData[i] >= minOccurance }
|
||||
if(len(wordData) == 0):
|
||||
# print "Added word: " + word
|
||||
wordData = {i: wordData[i] for i in wordData if wordData[i] >= minOccurance}
|
||||
if len(wordData) == 0:
|
||||
wordData = {"NOTHING": 1, "INTERESTING": 1, "TODAY": 1}
|
||||
with open(outfile + ".tmp", "w") as tmpFile:
|
||||
tmpFile.write(json.dumps(wordData))
|
||||
|
||||
+69
-26
@@ -10,27 +10,66 @@ import logging, sys
|
||||
|
||||
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Generate word cloud data based off of irc chat logs')
|
||||
parser.add_argument('-logfile', help='irc log file to read from', default='/home/archangelic/irc/log')
|
||||
parser.add_argument('-outfile', help='output file to write to', default='')
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate word cloud data based off of irc chat logs"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-logfile", help="irc log file to read from", default="/home/archangelic/irc/log"
|
||||
)
|
||||
parser.add_argument("-outfile", help="output file to write to", default="")
|
||||
|
||||
parser.add_argument('-timeend', type=int, help='end time of the word cloud (in epoch time)', default=calendar.timegm(time.gmtime()))
|
||||
parser.add_argument('-timestart', type=int, help='start time of the word cloud (in epoch time)', default=-1)
|
||||
parser.add_argument(
|
||||
"-timeend",
|
||||
type=int,
|
||||
help="end time of the word cloud (in epoch time)",
|
||||
default=calendar.timegm(time.gmtime()),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-timestart",
|
||||
type=int,
|
||||
help="start time of the word cloud (in epoch time)",
|
||||
default=-1,
|
||||
)
|
||||
|
||||
parser.add_argument('-bannedUsersFile', help='file containing list of banned users', default='/home/krowbar/Code/python/bannedUsers')
|
||||
parser.add_argument('-bannedWordsFile', help='file containing list of banned words', default='/home/krowbar/Code/python/bannedWords')
|
||||
parser.add_argument(
|
||||
"-bannedUsersFile",
|
||||
help="file containing list of banned users",
|
||||
default="/home/krowbar/Code/python/bannedUsers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-bannedWordsFile",
|
||||
help="file containing list of banned words",
|
||||
default="/home/krowbar/Code/python/bannedWords",
|
||||
)
|
||||
|
||||
parser.add_argument('-minLength', type=int, help='minimum size of words to include in the cloud', default=3)
|
||||
parser.add_argument('-minOccurrence', type=int, help='the minimum occurence of a word to include it in the cloud', default=3)
|
||||
parser.add_argument(
|
||||
"-minLength",
|
||||
type=int,
|
||||
help="minimum size of words to include in the cloud",
|
||||
default=3,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-minOccurrence",
|
||||
type=int,
|
||||
help="the minimum occurence of a word to include it in the cloud",
|
||||
default=3,
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
wordData = {} # keyed by "word" that contains a count
|
||||
#we only care about recent chats, let's say for the past sixteen hours
|
||||
wordData = {} # keyed by "word" that contains a count
|
||||
# we only care about recent chats, let's say for the past sixteen hours
|
||||
|
||||
args.timestart = args.timestart if args.timestart != -1 else args.timeend - (16 * 60 * 60)
|
||||
#timeCutoff = calendar.timegm(time.strptime("1 Oct 16", "%d %b %y"))
|
||||
logging.info("Generating word cloud based off words from " + str(args.timestart) + " to " + str(args.timeend))
|
||||
args.timestart = (
|
||||
args.timestart if args.timestart != -1 else args.timeend - (16 * 60 * 60)
|
||||
)
|
||||
# timeCutoff = calendar.timegm(time.strptime("1 Oct 16", "%d %b %y"))
|
||||
logging.info(
|
||||
"Generating word cloud based off words from "
|
||||
+ str(args.timestart)
|
||||
+ " to "
|
||||
+ str(args.timeend)
|
||||
)
|
||||
|
||||
bannedWords = open(args.bannedWordsFile).read().splitlines()
|
||||
bannedUsers = open(args.bannedUsersFile).read().splitlines()
|
||||
@@ -41,27 +80,31 @@ with open(args.logfile, "r") as log:
|
||||
time, user, message = line.split("\t", 3)
|
||||
time = int(time)
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
if user in bannedUsers:
|
||||
continue #We don't care what they say
|
||||
continue # We don't care what they say
|
||||
if time >= args.timestart and time <= args.timeend:
|
||||
#print "Processing line from " + user + " at " + str(time)
|
||||
for word in re.sub('[\'\"\`\/\\;:,.?!*&^\-()<>\{\}|_\[\]0-9]', ' ', message).lower().split():
|
||||
#changing symbols into spaces instead of stripping them avoids compounded words
|
||||
# print "Processing line from " + user + " at " + str(time)
|
||||
for word in (
|
||||
re.sub("['\"\`\/\\;:,.?!*&^\-()<>\{\}|_\[\]0-9]", " ", message)
|
||||
.lower()
|
||||
.split()
|
||||
):
|
||||
# changing symbols into spaces instead of stripping them avoids compounded words
|
||||
if len(word) < args.minLength or word in bannedWords:
|
||||
#print "Rejecting " + word
|
||||
# print "Rejecting " + word
|
||||
continue
|
||||
#if the word already exists in the list
|
||||
# if the word already exists in the list
|
||||
if word in wordData:
|
||||
wordData[word] += 1
|
||||
else: #if they are new
|
||||
else: # if they are new
|
||||
wordData[word] = 1
|
||||
#print "Added word: " + word
|
||||
wordData = {i:wordData[i] for i in wordData if wordData[i] >= args.minOccurrence }
|
||||
# print "Added word: " + word
|
||||
wordData = {i: wordData[i] for i in wordData if wordData[i] >= args.minOccurrence}
|
||||
if len(wordData) == 0:
|
||||
wordData = {"NOTHING": 1, "INTERESTING": 1, "TODAY": 1}
|
||||
if args.outfile == '':
|
||||
print json.dumps(wordData)
|
||||
if args.outfile == "":
|
||||
print(json.dumps(wordData))
|
||||
else:
|
||||
with open(args.outfile + ".tmp", "w") as tmpFile:
|
||||
tmpFile.write(json.dumps(wordData))
|
||||
|
||||
+29
-14
@@ -6,10 +6,13 @@ import calendar
|
||||
import shutil
|
||||
|
||||
logfile = "/home/archangelic/irc/log"
|
||||
#logfile = "/home/jumblesale/Code/irc/log"
|
||||
# logfile = "/home/jumblesale/Code/irc/log"
|
||||
outfile = "/home/krowbar/logs/chatStack.json"
|
||||
chatData = { 'hours': [], 'regions': {} } #hash keyed by "region" and then hour that counts chat instances
|
||||
#we only care about recent chats, let's say for the past couple weeks
|
||||
chatData = {
|
||||
"hours": [],
|
||||
"regions": {},
|
||||
} # hash keyed by "region" and then hour that counts chat instances
|
||||
# we only care about recent chats, let's say for the past couple weeks
|
||||
oneHour = 60 * 60
|
||||
oneWeek = 7 * 24 * 60 * 60
|
||||
timeNow = calendar.timegm(time.gmtime())
|
||||
@@ -17,38 +20,50 @@ timeCutoff = calendar.timegm(time.gmtime()) - (2 * oneWeek)
|
||||
|
||||
# this will eventually represent each region users are from
|
||||
def getAllRegions():
|
||||
return ['unknown']
|
||||
return ["unknown"]
|
||||
|
||||
|
||||
# this will provide a way to look up what region a user is from
|
||||
def getRegion(user):
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
|
||||
# populate the hours array with time 1 hour away from each other
|
||||
startTime = timeCutoff
|
||||
while startTime < timeNow:
|
||||
chatData['hours'].append(startTime)
|
||||
chatData["hours"].append(startTime)
|
||||
startTime += oneHour
|
||||
|
||||
# populate the regions array with blank data for each region
|
||||
for region in getAllRegions():
|
||||
chatData['regions'][region] = { 'name': region, 'values': [0] * len(chatData['hours']) }
|
||||
chatData["regions"][region] = {
|
||||
"name": region,
|
||||
"values": [0] * len(chatData["hours"]),
|
||||
}
|
||||
|
||||
with open(logfile, "r") as log:
|
||||
hourIdx = 0 # starting with the oldest time slot, we will count instances of user chatting
|
||||
hourIdx = (
|
||||
0
|
||||
) # starting with the oldest time slot, we will count instances of user chatting
|
||||
for line in log:
|
||||
try:
|
||||
time, user, message = line.split("\t", 3)
|
||||
time = int(time)
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
if time > timeCutoff:
|
||||
region = getRegion(user)
|
||||
while time > chatData['hours'][hourIdx] + oneHour: # we are past the current hour idx, move ahead until we find the right idx
|
||||
hourIdx += 1;
|
||||
if hourIdx >= len(chatData['hours']):
|
||||
break; #uh oh! somehow we are parsing a line from the future! we're in pretty bad shape!
|
||||
while (
|
||||
time > chatData["hours"][hourIdx] + oneHour
|
||||
): # we are past the current hour idx, move ahead until we find the right idx
|
||||
hourIdx += 1
|
||||
if hourIdx >= len(chatData["hours"]):
|
||||
break
|
||||
# uh oh! somehow we are parsing a line from the future! we're in pretty bad shape!
|
||||
# hourIdx should now be a good value
|
||||
chatData['regions'][region]['values'][hourIdx] += 1 # increment the user region's count for the current hour
|
||||
chatData["regions"][region]["values"][
|
||||
hourIdx
|
||||
] += 1 # increment the user region's count for the current hour
|
||||
|
||||
with open(outfile + ".tmp", "w") as tmpFile:
|
||||
tmpFile.write(json.dumps(chatData))
|
||||
|
||||
+60
-56
@@ -7,25 +7,25 @@ import shutil
|
||||
import re
|
||||
|
||||
logfile = "/home/archangelic/irc/log"
|
||||
#logfile = "/home/jumblesale/Code/irc/log"
|
||||
# logfile = "/home/jumblesale/Code/irc/log"
|
||||
outfile = "/home/krowbar/logs/chatStats.json"
|
||||
userData = {} #hash keyed by "user" that contains a start timestamp, last timestamp, last said string, chat count, letter count, and word count
|
||||
#also now happy emotes and sad emotes
|
||||
userData = {} # hash keyed by "user" that contains a start timestamp, last timestamp, last said string, chat count, letter count, and word count
|
||||
# also now happy emotes and sad emotes
|
||||
rejectRegexp = "http[s]?://|[0-9]{2}[;:][0-9]{2}"
|
||||
happyRegexp = ":[-]?[])}]"
|
||||
sadRegexp = ":[-]?[[({]"
|
||||
nameFix = {
|
||||
'archangel': 'archangelic',
|
||||
'jumblesal': 'jumblesale',
|
||||
'hardmath1': 'kc',
|
||||
'hardmath123': 'kc',
|
||||
'bendorphan': 'endorphant',
|
||||
'endorphan': 'endorphant',
|
||||
'synergian': 'synergiance'
|
||||
}
|
||||
"archangel": "archangelic",
|
||||
"jumblesal": "jumblesale",
|
||||
"hardmath1": "kc",
|
||||
"hardmath123": "kc",
|
||||
"bendorphan": "endorphant",
|
||||
"endorphan": "endorphant",
|
||||
"synergian": "synergiance",
|
||||
}
|
||||
|
||||
with open(logfile, "r") as log:
|
||||
lastUser = "";
|
||||
lastUser = ""
|
||||
currentStreak = 1
|
||||
for line in log:
|
||||
try:
|
||||
@@ -36,66 +36,70 @@ with open(logfile, "r") as log:
|
||||
else:
|
||||
user = user.lower()
|
||||
except ValueError:
|
||||
continue #There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
continue # There are some bad lines in the log file that we'll ignore if we can't parse
|
||||
if user in userData:
|
||||
day = time / 86400 #seconds in a day
|
||||
if userData[user]['startTime'] == 0:
|
||||
userData[user]['startTime'] = time
|
||||
if userData[user]['lastDay'] != day:
|
||||
userData[user]['daysActive'] += 1
|
||||
userData[user]['lastDay'] = day
|
||||
day = time / 86400 # seconds in a day
|
||||
if userData[user]["startTime"] == 0:
|
||||
userData[user]["startTime"] = time
|
||||
if userData[user]["lastDay"] != day:
|
||||
userData[user]["daysActive"] += 1
|
||||
userData[user]["lastDay"] = day
|
||||
if lastUser == user:
|
||||
currentStreak += 1
|
||||
if currentStreak > userData[user]['streak']:
|
||||
userData[user]['streak'] = currentStreak
|
||||
if currentStreak > userData[user]["streak"]:
|
||||
userData[user]["streak"] = currentStreak
|
||||
else:
|
||||
currentStreak = 1
|
||||
if userData[user]['lastMention']:
|
||||
resTime = time - userData[user]['lastMention']
|
||||
userData[user]['responseTime'] += min(resTime, 7200) #cap the value at 2hrs, things get skewed toward the high end otherwise
|
||||
userData[user]['lastMention'] = 0
|
||||
if userData[user]["lastMention"]:
|
||||
resTime = time - userData[user]["lastMention"]
|
||||
userData[user]["responseTime"] += min(
|
||||
resTime, 7200
|
||||
) # cap the value at 2hrs, things get skewed toward the high end otherwise
|
||||
userData[user]["lastMention"] = 0
|
||||
|
||||
userData[user]['endTime'] = time
|
||||
#userData[user]['lastSaid'] = message
|
||||
userData[user]['lineCount'] += 1
|
||||
userData[user]['wordCount'] += len(message.split())
|
||||
userData[user]['charCount'] += len(message)
|
||||
else: #if they are new
|
||||
userData[user]["endTime"] = time
|
||||
# userData[user]['lastSaid'] = message
|
||||
userData[user]["lineCount"] += 1
|
||||
userData[user]["wordCount"] += len(message.split())
|
||||
userData[user]["charCount"] += len(message)
|
||||
else: # if they are new
|
||||
userData[user] = {}
|
||||
userData[user]['startTime'] = time
|
||||
userData[user]['endTime'] = time
|
||||
#userData[user]['lastSaid'] = message
|
||||
userData[user]['lineCount'] = 1
|
||||
userData[user]['wordCount'] = len(message.split())
|
||||
userData[user]['charCount'] = len(message)
|
||||
userData[user]['daysActive'] = 1
|
||||
userData[user]['lastDay'] = time / 86400
|
||||
userData[user]['streak'] = 1
|
||||
userData[user]['mentions'] = 0
|
||||
userData[user]['lastMention'] = 0
|
||||
userData[user]['responseTime'] = 0
|
||||
userData[user]['botUse'] = 0
|
||||
userData[user]['happyEmotes'] = 0
|
||||
userData[user]['sadEmotes'] = 0
|
||||
userData[user]["startTime"] = time
|
||||
userData[user]["endTime"] = time
|
||||
# userData[user]['lastSaid'] = message
|
||||
userData[user]["lineCount"] = 1
|
||||
userData[user]["wordCount"] = len(message.split())
|
||||
userData[user]["charCount"] = len(message)
|
||||
userData[user]["daysActive"] = 1
|
||||
userData[user]["lastDay"] = time / 86400
|
||||
userData[user]["streak"] = 1
|
||||
userData[user]["mentions"] = 0
|
||||
userData[user]["lastMention"] = 0
|
||||
userData[user]["responseTime"] = 0
|
||||
userData[user]["botUse"] = 0
|
||||
userData[user]["happyEmotes"] = 0
|
||||
userData[user]["sadEmotes"] = 0
|
||||
|
||||
lastUser = user;
|
||||
if message.rstrip() and message[0] == '!':
|
||||
userData[user]['botUse'] += 1
|
||||
lastUser = user
|
||||
if message.rstrip() and message[0] == "!":
|
||||
userData[user]["botUse"] += 1
|
||||
if not re.search(rejectRegexp, message):
|
||||
if re.search(happyRegexp, message):
|
||||
userData[user]['happyEmotes'] += 1
|
||||
userData[user]["happyEmotes"] += 1
|
||||
if re.search(sadRegexp, message):
|
||||
userData[user]['sadEmotes'] += 1
|
||||
userData[user]["sadEmotes"] += 1
|
||||
|
||||
try:
|
||||
if message.rstrip() and message.split()[0][-1] == ':': #last character of first word
|
||||
if (
|
||||
message.rstrip() and message.split()[0][-1] == ":"
|
||||
): # last character of first word
|
||||
name = message.split()[0][0:-1]
|
||||
if user != name and userData.has_key(name):
|
||||
userData[name]['mentions'] += 1
|
||||
if not userData[name]['lastMention']:
|
||||
userData[name]['lastMention'] = time
|
||||
userData[name]["mentions"] += 1
|
||||
if not userData[name]["lastMention"]:
|
||||
userData[name]["lastMention"] = time
|
||||
except IndexError:
|
||||
print '##' + message + '##'
|
||||
print("##" + message + "##")
|
||||
continue
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import urllib
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def get_track():
|
||||
pass
|
||||
+35
-16
@@ -4,26 +4,45 @@ import random
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Print some random dictionary words.')
|
||||
parser.add_argument('-d', dest='dictionary', metavar='DICT',
|
||||
help='supply a dictionary', default='/usr/share/dict/american-english-large')
|
||||
parser.add_argument('-c', dest='count', type=int,
|
||||
help='specify how many words you want per line', default=10)
|
||||
parser.add_argument('-l', dest='lines', type=int,
|
||||
help='specify how many lines of random words you want', default=1)
|
||||
parser.add_argument('--no-appos', action='store_true',
|
||||
help='remove words with appostrophes')
|
||||
parser.add_argument('--no-proper', action='store_true',
|
||||
help='remove words that start with a capital letter')
|
||||
parser = argparse.ArgumentParser(description="Print some random dictionary words.")
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
dest="dictionary",
|
||||
metavar="DICT",
|
||||
help="supply a dictionary",
|
||||
default="/usr/share/dict/american-english-large",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
dest="count",
|
||||
type=int,
|
||||
help="specify how many words you want per line",
|
||||
default=10,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
dest="lines",
|
||||
type=int,
|
||||
help="specify how many lines of random words you want",
|
||||
default=1,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-appos", action="store_true", help="remove words with appostrophes"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-proper",
|
||||
action="store_true",
|
||||
help="remove words that start with a capital letter",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
#print args
|
||||
# print args
|
||||
|
||||
with open(args.dictionary, "r") as wordsfile:
|
||||
words = wordsfile.readlines()
|
||||
if(args.no_appos):
|
||||
words[:] = [word for word in words if word.find('\'') == -1]
|
||||
if(args.no_proper):
|
||||
if args.no_appos:
|
||||
words[:] = [word for word in words if word.find("'") == -1]
|
||||
if args.no_proper:
|
||||
words[:] = [word for word in words if not word[0].isupper()]
|
||||
for _ in range(0, args.lines):
|
||||
print ' '.join([w.strip("\n") for w in random.sample(words, args.count)])
|
||||
print(" ".join([w.strip("\n") for w in random.sample(words, args.count)]))
|
||||
Reference in new issue
Block a user