Periodic git commit

Code/python/bannedWords

@@ -197,3 +197,34 @@ there
 doing
 even
 does
+theres
+theyve
+were
+using
+could
+should
+wont
+way
+though
+being
+while
+cant
+only
+well
+very
+didnt
+hes
+quite
+havent
+weant
+might
+say
+mean
+still
+come
+will
+used
+want
+roll
+dice
+length

Code/python/chatchecker.py

@@ -8,8 +8,10 @@ import shutil
 logfile = "/home/jumblesale/Code/irc/log"
 outfile = "/home/krowbar/logs/userData.json"
 userData = {} #hash keyed by "user" that contains an array of timestamps
-#we only care about recent chats, let's say for the past two weeks
-timeCutoff = calendar.timegm(time.gmtime()) - (2 * 7 * 24 * 60 * 60)
+#we only care about recent chats, let's say for the past couple weeks
+oneWeek = 7 * 24 * 60 * 60
+fiveMinutes = 5 * 60
+timeCutoff = calendar.timegm(time.gmtime()) - (2 * oneWeek)
 
 with open(logfile, "r") as log:
     for line in log:
@@ -19,8 +21,12 @@ with open(logfile, "r") as log:
         except ValueError:
             continue #There are some bad lines in the log file that we'll ignore if we can't parse
         if time > timeCutoff:
+            #standardize all times into a single "week"
+            #time %= oneWeek
             #if the user already exists in the list
             if user in userData:
+                if time < userData[user][-1] + fiveMinutes:
+                    continue #only record times that are at least five minutes apart
                 userData[user].append(time) #append the new time
             else: #if they are new
                 userData[user] = [time] #create a new list

Code/python/chatcloud.py

@@ -8,12 +8,17 @@ import shutil
 
 logfile = "/home/jumblesale/Code/irc/log"
 outfile = "/home/krowbar/logs/chatcloud.json"
+bannedUsersFile = "/home/krowbar/Code/python/bannedUsers"
 bannedWordsFile = "/home/krowbar/Code/python/bannedWords"
 wordData = {} # keyed by "word" that contains a count
-#we only care about recent chats, let's say for the past two weeks
-timeCutoff = calendar.timegm(time.gmtime()) - (2 * 7 * 24 * 60 * 60)
-minOccurance = 10
+#we only care about recent chats, let's say for the past sixteen hours
+timeTo = calendar.timegm(time.gmtime())
+timeCutoff = timeTo - (16 * 60 * 60)
+print "Generating word cloud based off words from " + str(timeCutoff) + " to " + str(timeTo)
+minOccurance = 3 #we'll have to reduce the minOccurances if we reduce the timeCutoff
+minLength = 3 #number of letters long
 bannedWords = open(bannedWordsFile).read().splitlines()
+bannedUsers = open(bannedUsersFile).read().splitlines()
 
 with open(logfile, "r") as log:
     for line in log:
@@ -22,9 +27,13 @@ with open(logfile, "r") as log:
             time = int(time)
         except ValueError:
             continue #There are some bad lines in the log file that we'll ignore if we can't parse
-        if time > timeCutoff:
-            for word in re.sub('[\'\";:,.?!*&^\-()\[\]]', '', message).lower().split():
-                if word in bannedWords:
+        if user in bannedUsers:
+            continue #We don't care what they say
+        if time >= timeCutoff and time <= timeTo:
+            #print "Processing line from " + user + " at " + str(time)
+            for word in re.sub('[\'\"\`\/\\;:,.?!*&^\-()<>|_\[\]0-9]', '', message).lower().split():
+                if len(word) < minLength or word in bannedWords:
+                    #print "Rejecting " + word
                     continue
                 #if the word already exists in the list
                 if word in wordData:
@@ -33,6 +42,8 @@ with open(logfile, "r") as log:
                     wordData[word] = 1
                     #print "Added word: " + word
 wordData = {i:wordData[i] for i in wordData if wordData[i] >= minOccurance }
+if(len(wordData) == 0):
+    wordData = {"NOTHING": 1, "INTERESTING": 1, "TODAY": 1}
 with open(outfile + ".tmp", "w") as tmpFile:
     tmpFile.write(json.dumps(wordData))
 shutil.move(outfile + ".tmp", outfile)