stopwords 2.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Copyright (C) 2016 Constant, Algolit
  4. # This program is free software: you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation, either version 3 of the License, or
  7. # (at your option) any later version.
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details: <http://www.gnu.org/licenses/>.
  12. '''
  13. Input texts are checked against occurences of certain words included in a list of "stopwords" established by NLTK (Natural Language Toolkit). These words are then removed.
  14. In data mining, text processing and machine learning, these so-called high frequency words are filtered out before or after natural language data is processed.
  15. Relational words such as 'the', 'is', 'at', 'which', and 'on' are considered redundant because they are too frequent, and meaningless once the word order is removed.
  16. '''
  17. # A set is a list with unique words
  18. hyperlien = set()
  19. # define list of filtered words
  20. filtered_words = []
  21. # read stopwords from file & save them in a list
  22. # read from file
  23. # txt = source.readlines()
  24. with open("english.txt", "r") as source:
  25. # for each line
  26. for line in source:
  27. # clean returns
  28. line = line.strip()
  29. # add word to set stopwords (cfr difference with list: list.append())
  30. hyperlien.add(line)
  31. # define your sentence / string
  32. sentence = 'I was at Synesthésie last night and took a bus to go home.'
  33. # print sentence
  34. print("phrase originale:", sentence)
  35. # convert string to list of words
  36. words = sentence.split(" ")
  37. # for each word of list, check if word is in stopwords, if it isn't, add word to filtered wordlist
  38. for word in words:
  39. if word not in hyperlien:
  40. filtered_words.append(word)
  41. # this is the same, but shorter + no need to declare filtered_words as list in the beginning:
  42. #filtered_words = [word for word in words if word not in stopwords]
  43. # turn wordlist into string of characters
  44. new_sentence = " ".join(filtered_words)
  45. # print new sentence
  46. print("phrase réécrite:", new_sentence)