commited all contents created by participants

content/adrien_chacon/sources/2_toolbox/adapting_the_reading_glasses.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env/ python
+# THIS SCRIPT ADAPTS THE SOURCE TEXT IN THE RIGHT READING FORMAT FOR THE ALGORITHM, CLEANING UP WHITE SPACES/SPLITTING INTO SENTENCES
+# source text is written in uppercase
+# remove white spaces, put everything in lowercase
+# split on punctuation
+# write in file capitalizing first letter
+
+ #    Copyright (C) 2016 Constant, Algolit, An Mertens
+
+ #    This program is free software: you can redistribute it and/or modify
+ #    it under the terms of the GNU General Public License as published by
+ #    the Free Software Foundation, either version 3 of the License, or
+ #    (at your option) any later version.
+
+ #    This program is distributed in the hope that it will be useful,
+ #    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ #    GNU General Public License for more details: <http://www.gnu.org/licenses/>.
+
+
+import nltk.data
+
+# split into sentences
+sentences = []
+finding_sentences = nltk.data.load('tokenizers/punkt/english.pickle')
+with open('../data/frankenstein_for_machines.txt', 'rt') as source:
+	for line in source:
+		# this returns a list with 1 element containing the entire text, sentences separated by \n
+		sentences = '\n'.join(finding_sentences.tokenize(line.strip().lower().capitalize()))
+		# transform string into list of sentences
+		sentences = sentences.split("\n")
+
+# write clean text to a file
+with open("frankenstein_for_machines_tf.txt", "w") as destination:
+	for sentence in sentences:
+		destination.write(sentence.strip().capitalize()+" ")
+
+

content/adrien_chacon/sources/2_toolbox/counting_a_frequency_of_words.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env/ python
+
+# This script creates a sorted frequency dictionary with stopwords.
+
+ #    Copyright (C) 2016 Constant, Algolit, An Mertens
+
+ #    This program is free software: you can redistribute it and/or modify
+ #    it under the terms of the GNU General Public License as published by
+ #    the Free Software Foundation, either version 3 of the License, or
+ #    (at your option) any later version.
+
+ #    This program is distributed in the hope that it will be useful,
+ #    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ #    GNU General Public License for more details: <http://www.gnu.org/licenses/>.
+
+from __future__ import division
+from collections import Counter
+import string
+from nltk.corpus import stopwords
+
+
+# VARIABLES
+
+
+# textfiles
+source1 = open('../data/1984_fragment.txt', 'rt')
+source2 = open('../data/verne_fragment.txt', 'rt')
+destination1 = open('../data/counting_1984.txt', 'wt')
+destination2 = open('../data/counting_verne.txt', 'wt')
+
+
+# FUNCTIONS
+
+# PREPROCESSING TEXT FILE
+## remove caps + breaks + punctuation
+def remove_punct(f):
+	tokens = (' '.join(line.replace('\n', '') for line in f)).lower()
+	for c in string.punctuation:
+		tokens= tokens.replace(c,"")
+		tokens = tokens.strip()
+		#print("tokens", type(tokens))
+	return tokens
+
+## create frequency dictionary 
+def freq_dict(tokens):
+	tokens = tokens.split(" ")
+	frequency_d = {}
+	# tokens = tokens.split(" ")
+	for token in tokens:
+		try:
+			frequency_d[token] += 1
+		except KeyError:
+			frequency_d[token] = 1
+	return frequency_d
+
+## sort words by frequency (import module)
+def sort_dict(frequency_d):
+	c=Counter(frequency_d)
+	frequency = c.most_common()
+	return frequency
+
+# write words in text file 
+def write_to_file(frequency, g):
+	for key, value in frequency:
+		g.write(("{} : {} \n".format(key, value)))
+	g.close()
+
+
+# Write new text into logbook
+def writetologbook(content):
+    try:
+        log = open(filename, "a")
+        try:
+            log.write(content)
+        finally:
+            log.close()
+    except IOError:
+        pass
+
+
+# SCRIPT
+
+# execute functions
+
+tokens1 = remove_punct(source1)
+tokens2 = remove_punct(source2)
+
+frequency_d1 = freq_dict(tokens1)
+frequency_d2 = freq_dict(tokens2)
+
+frequency1 = sort_dict(frequency_d1)
+frequency2 = sort_dict(frequency_d2)
+
+# Write in textfile
+
+write_to_file(frequency1, destination1)
+write_to_file(frequency2, destination2)
+
+source1.close()
+source2.close()
+
+destination1.close()
+destination2.close()
+
+
+

content/adrien_chacon/sources/2_toolbox/counting_a_frequency_of_words_grand_cru.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env/ python
+
+# This script creates a sorted frequency dictionary with stopwords
+
+ #    Copyright (C) 2016 Constant, Algolit, An Mertens
+
+ #    This program is free software: you can redistribute it and/or modify
+ #    it under the terms of the GNU General Public License as published by
+ #    the Free Software Foundation, either version 3 of the License, or
+ #    (at your option) any later version.
+
+ #    This program is distributed in the hope that it will be useful,
+ #    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ #    GNU General Public License for more details: <http://www.gnu.org/licenses/>.
+
+
+
+from collections import Counter
+import string
+import nltk
+
+
+
+'''
+This script creates a frequency dictionary of words used in a text filtering out stopwords
+'''
+
+# VARIABLES
+
+
+
+# textfiles
+source1 = open('../data/1984_fragment.txt', 'rt')
+source2 = open('../data/verne_fragment.txt', 'rt')
+destination1 = open('../data/grand_cru_counting_1984.txt', 'wt')
+destination2 = open('../data/grand_cru_counting_verne.txt', 'wt')
+
+
+freqwords = ["the", "a", "to", "of", "in", 'is', "with", "on", "for", "at", "from", "about",\
+ "are", "an", "up", "out", "have", "be", "this", "one", "says", "as", "all", "just", "was", "so", "there", "not", "by",\
+ "into", "been", "dont", "has", "over", "doesnt", "did", "had", "would", "could", "didnt"]
+relationals = ["she", "you", "i", "he", "we", "her", "his", "it", "its", "their", "me", "our", 'they', "us", "my",\
+"your", "theyre", 'them', "youre", "him", "were", "these"]
+subphrases = ["and", "that", "but", "like", "what", "if", "then","theres", "or", "which", "who", "while", "where", "when",\
+"thats", "how", "because"]
+
+# removed "she", "you", "i", "he", "we",
+stopwords = ["the", "a", "to", "of", "in", 'is', "with", "on", "for", "at", "from", "about",\
+ "are", "an", "up", "out", "have", "be", "this", "one", "says", "as", "all", "just", "was", "so",\
+ "her", "his", "it", "its", "their", "me", "our",\
+ "and", "that", "but", "like", "what", "if", "then", "there", "they", "us", "my", "your", "theres", "theyre", "or", "not",\
+ "which", "by", "who", "them", "into", "while", "been", "dont", "where", "youre", "has", "when", "over", "him", "were", "doesnt",\
+ "did", "thats", "how", "had", "these", "would", "could", "because", "didnt"]
+
+
+## FUNCTIONS
+
+# PREPROCESSING TEXT FILE
+## remove caps + breaks + punctuation
+def remove_punct(f):
+	tokens = (' '.join(line.replace('\n', '') for line in f)).lower()
+	for c in string.punctuation:
+		tokens= tokens.replace(c,"")
+		tokens = tokens.strip()
+		#print("tokens", type(tokens))
+	return tokens
+
+# remove stopwords
+def remove_stopwords(tokens):
+	tokens = tokens.split(" ")
+	words =[]
+	for token in tokens:
+		if token not in stopwords:
+			words.append(token)
+	return words
+
+## create frequency dictionary 
+def freq_dict(words):
+	frequency_d = {}
+	# tokens = tokens.split(" ")
+	for word in words:
+		try:
+			frequency_d[word] += 1
+		except KeyError:
+			frequency_d[word] = 1
+	return frequency_d
+
+## sort words by frequency (import module)
+def sort_dict(frequency_d):
+	c=Counter(frequency_d)
+	frequency = c.most_common()
+	return frequency
+
+# write words in text file 
+def write_to_file(frequency, destination):
+	for key, value in frequency:
+		destination.write(("{} : {} \n".format(key, value)))
+	destination.close()
+
+
+# Write new text into logbook
+def writetologbook(content):
+    try:
+        log = open(filename, "a")
+        try:
+            log.write(content)
+        finally:
+            log.close()
+    except IOError:
+        pass
+
+
+## SCRIPT
+
+# execute functions
+tokens1 = remove_punct(source1)
+tokens2 = remove_punct(source2)
+
+words1 = remove_stopwords(tokens1)
+words2 = remove_stopwords(tokens2)
+
+frequency_d1 = freq_dict(words1)
+frequency_d2 = freq_dict(words2)
+
+frequency1 = sort_dict(frequency_d1)
+frequency2 = sort_dict(frequency_d2)
+
+write_to_file(frequency1, destination1)
+write_to_file(frequency2, destination2)
+
+source1.close()
+source2.close()
+
+destination1.close()
+destination2.close()
+
+# -------------------------------------------

content/adrien_chacon/sources/2_toolbox/most_frequent_trigrams.py

@@ -0,0 +1,54 @@
+from collections import Counter
+import nltk
+import re
+import pickle
+
+
+# VARIABLES
+
+source = open("../data/1984_fragment.txt", "r")
+destination = open("../data/1984_fragment_trigrams.txt", "w")
+destination.write("OBAMA\S MOST FREQUENT TRIGRAMS with Penn's TREEBANK\n\n\n")
+
+
+# FUNCTIONS
+## sort words by frequency (import module)
+def sort_dict(frequency_d):
+	c=Counter(frequency_d)
+	frequency = c.most_common()
+	return frequency
+
+## MAKE SURE ALL VARIABLES ARE DECLARED WITHIN THE LOOPS		
+
+# 1. Create dictionary of trigrams
+trigrams = {}
+for line in source:
+	# remove punctuation
+	clean_tri = []
+	words = line.split(" ")
+	for word in words:
+		cleaning = re.compile(r"[A-Za-z0-9]")
+		if cleaning.match(word):
+			clean_tri.append(word)
+		else:
+			pass
+	# find trigrams
+	tricount = nltk.trigrams(clean_tri)	
+	# count frequency of each trigram and add trigram + value in dictionary			
+	for trigram in tricount:
+		if trigram in trigrams:
+			trigrams[trigram] += 1
+		else:
+			trigrams[trigram] = 1	
+
+trigrams_sorted = sort_dict(trigrams)
+first10pairs = trigrams_sorted[:10]
+
+
+with destination as text:
+	for tri, frequency in first10pairs:
+		text.write("{} : {} \n".format(tri, frequency))
+
+
+
+

content/adrien_chacon/sources/2_toolbox/python2_write_parts_of_speech.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# this script works in Python2
+
+from __future__ import division
+import nltk
+from pattern.en import tag
+import nltk.data
+from random import shuffle, choice
+
+
+# VARIABLES
+
+
+# texts
+source = open("../data/1984_fragment.txt", "r")
+destination = open("../data/1984_fragment_pos.txt", "wt")
+destination.write("1984\S SYNTAX using PENN'S TREEBANK\n\n")
+
+
+
+# FUNCTIONS
+
+## SCRIPT
+
+# select 1 or more sentences from source
+## split source text into list of sentences
+finding_sentences = nltk.data.load('tokenizers/punkt/english.pickle')
+sentences_list = []
+with source as text0:
+    for line in text0:
+        # this returns a list with 1 element containing the entire text, sentences separated by \n
+        sentences = '\n'.join(finding_sentences.tokenize(line.decode('utf-8').strip()))
+        # transform string into list of sentences
+        sentences_list = sentences.split("\n") 
+        print("sentences list", sentences_list)
+
+with destination as text1:
+	for sentence in sentences_list:
+	# create tuple of tuples with pairs of word + POS-tag
+		collection = tag(sentence, tokenize=True, encoding='utf-8')
+	# transform tuple into list to be able to manipulate it
+		collection = list(collection)
+		for element in collection:
+			text1.write(element[1] + " ")
+	

content/adrien_chacon/sources/2_toolbox/remove_digits.py

@@ -0,0 +1,15 @@
+import re
+
+
+source = open("../data/1984_fragment.txt", "r")
+#source = open("joyce.txt", "r")
+destination = open("../data/1984_no_digits.txt", "w")
+
+
+sentences = []
+for line in source:
+	result = ''.join([i for i in line if not i.isdigit()])
+	destination.write(result)
+
+source.close()
+destination.close()

content/adrien_chacon/sources/2_toolbox/resize_imgbatch_animated_gif.txt

@@ -0,0 +1,40 @@
+
+animated gif
+_____________
+
+# all images have width of 360 and keep aspect ratio, are overwritten
+mogrify '*.JPG[360x]'
+# all images have width of 360 and keep aspect ratio, are renamed
+convert '*.jpg[200x]' resized%03d.png
+
+mogrify -resize 640x480 *.JPG
+convert -delay 140 -loop 0 *.JPG uitnodiging.gif
+convert -delay 200 0.JPG 1.JPG 2.JPG 3.JPG 4.JPG 5.JPG 6.JPG 7.JPG 8.JPG 9.JPG 10.JPG 11.JPG 12.JPG 13.JPG 14.JPG 15.JPG 16.JPG 17.JPG 18.JPG 19.JPG 21.JPG 22.JPG 23.JPG 24.JPG 25.JPG 26.JPG 27.JPG 28.JPG 29.JPG 30.JPG 31.JPG 32.JPG 33.JPG 34.JPG -loop 0 uitnodiging.gif
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Ssssssssssst!' label0.JPG
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Top Secret!' 1_label7.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Laatste Tuinfeest' label1.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Verrassing voor\n\nJuliette & Michel' 1_label8.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'28 mei 2017\n\n15u' label1.JPG label3.JPG
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Houtseweg 33\n\n2340 Beerse' label4.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 42 -tile pattern:checkerboard -size 640x480 -gravity center label:'Het huis is leeg.\n\n Water en elektriciteit\n zijn afgesloten.' label5.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 42 -tile pattern:checkerboard -size 640x480 -gravity center label:'Daar hebben wij,\n de kinderen,\n oplossingen voor gevonden.' label6.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 42 -tile pattern:checkerboard -size 640x480 -gravity center label:'Wij zorgen voor\n een hapje en een drankje.' label7.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 42 -tile pattern:checkerboard -size 640x480 -gravity center label:'Brengen jullie graag mee:\n\n een klapstoel,\n een goed humeur,\n herinneringen aan het huis.' label8.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 42 -tile pattern:checkerboard -size 640x480 -gravity center label:'En potjes aarde,\n\n als je een plantje\n uit de tuin wil.' label9.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Graag\nbevestiging\n voor 15 mei' label10.JPG
+
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Hopelijk\ntot dan!' 28.JPG
+
+convert -background lime -fill green  -font AvantGarde-Book -pointsize 72 -tile pattern:checkerboard -size 640x480 -gravity center label:'Erik, Jan, \n An, Geert, \n Olivia, Giulia' label12.JPG
+