content.generator.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. # @Author: Bachir Soussi Chiadmi <bach>
  4. # @Date: 27-03-2017
  5. # @Email: bachir@figureslibres.io
  6. # @Last modified by: bach
  7. # @Last modified time: 21-04-2017
  8. # @License: GPL-V3
  9. import sys, os, shutil
  10. import markdown
  11. # import mistune
  12. from bs4 import BeautifulSoup
  13. import pypandoc
  14. import json
  15. import re
  16. import pyphen
  17. _BOOK_SRC = 'book-src'
  18. _BUILD_d = "build"
  19. # CUR_PATH = os.path.dirname(os.path.abspath(__file__))
  20. # hyphenator
  21. _H_FR = pyphen.Pyphen(lang='fr')
  22. _ERROR_PREF = '\033[1m[!!]\033[0m '
  23. print("Building book")
  24. def main():
  25. # clean build directory
  26. if os.path.isdir(_BUILD_d):
  27. shutil.rmtree(_BUILD_d, ignore_errors=True)
  28. os.mkdir(_BUILD_d)
  29. print('Hyphen has fr language')
  30. print('fr' in pyphen.LANGUAGES)
  31. parse_book(_BOOK_SRC)
  32. def parse_book(book):
  33. # book_name = book.replace('.git', '')
  34. # print("- - -")
  35. print("Parse book")
  36. # print("- - -")
  37. # table of content (ordered list of markdown files)
  38. sum_p = os.path.join(_BOOK_SRC, "SUMMARY.md")
  39. if not os.path.isfile(sum_p):
  40. print("No summary file, can't generate html")
  41. return
  42. sum_f = open(sum_p)
  43. sum_str = sum_f.read()
  44. # print(sum_str)
  45. # convert md to html
  46. sum_html = markdown.markdown(sum_str)
  47. # print(sum_html)
  48. # create dom from html string (as it will be parsable)
  49. sum_dom = BeautifulSoup(sum_html, 'html.parser')
  50. # print(sum_dom)
  51. # parse html dom to get file list in the right order
  52. toc = parse_summary(sum_dom.ul, {})
  53. # print(toc)
  54. # generate final html build for html2print
  55. generate_html(book, toc)
  56. def parse_summary(ul, toc):
  57. print("Parse summary")
  58. i=0
  59. for li in ul.find_all('li',recursive=False):
  60. # print('li')
  61. for a in li.find_all('a',recursive=False):
  62. # print(a.get_text(strip=True))
  63. # print(a['href'])
  64. href = a['href']
  65. href = re.sub(r'^/', '', href)
  66. toc[i] = {
  67. 'label':a.get_text(strip=True),
  68. 'file':href
  69. }
  70. i = i+1
  71. return toc
  72. def generate_html(book, toc):
  73. print("Generate html")
  74. #
  75. # create main html dom from template
  76. template_f = open("templates/main.tpl.html", "r")
  77. template_html = template_f.read()
  78. template_dom = BeautifulSoup(template_html, 'html.parser')
  79. # replace title
  80. # template_dom.html.head.title.contents[0].replaceWith(book_name)
  81. # get story div
  82. story_dom = template_dom.find('div', {"id":"flow-main"})
  83. # loop through pages to convert them to html and add it to main html file
  84. pi = 0
  85. for page in toc:
  86. # print(toc[p])
  87. pagename = toc[page]['label']
  88. pageid = re.sub('[^a-z0-9]+', '-', pagename.lower())
  89. print("\033[92m"+pageid+"\033[0m")
  90. # files
  91. in_f = os.path.join(_BOOK_SRC, toc[page]['file'])
  92. if not os.path.isfile(in_f):
  93. print(_ERROR_PREF+"Source path is not a file, can't generate html : "+in_f)
  94. continue
  95. # print('in_f : '+in_f)
  96. # out_f = os.path.join(book_build_d_pages, toc[p]['file'].replace('/', '-').replace('.md', '.html'))
  97. # print('out_f : '+out_f)
  98. pdoc_args = ['--mathjax',
  99. '--smart']
  100. pdoc_filters = []
  101. output = pypandoc.convert_file(in_f,
  102. to='html5',
  103. format='markdown+header_attributes+link_attributes+bracketed_spans',
  104. extra_args=pdoc_args,
  105. filters=pdoc_filters)
  106. # outputfile=out_f)
  107. # print("output :\n"+output)
  108. output_dom = BeautifulSoup(output, 'html.parser')
  109. # print("output_dom :")
  110. # print(output_dom)
  111. # hyphenate paragraphes
  112. for node in output_dom.find_all('p'):
  113. hyphenate(node)
  114. # print(str(output_dom))
  115. # copy images
  116. for img in output_dom.find_all('img'):
  117. # print('-- img ',img)
  118. att_src = re.sub(r"^\/", "", img['src'])
  119. img['src'] = att_src
  120. # domimg = output_dom.find('img', {'src':img['src']})
  121. # domimg['src'] = att_src
  122. # print(domimg)
  123. src_img = os.path.join(_BOOK_SRC, att_src)
  124. # print('- - '+src_img)
  125. if not os.path.isfile(src_img):
  126. print(_ERROR_PREF+"Source path is not a file, can't copy img : \033[1m"+src_img+"\033[0m")
  127. continue
  128. dest_img = os.path.join(_BUILD_d, att_src)
  129. # print('- - '+dest_img)
  130. dest_path, dest_file = os.path.split(dest_img)
  131. if not os.path.isdir(dest_path):
  132. os.makedirs(dest_path)
  133. shutil.copyfile(src_img, dest_img)
  134. # append html story page to template_dom
  135. story_page = BeautifulSoup('<div class="story-page story-page-'+str(pi)+'" id="'+pageid+'"></div>', 'html.parser')
  136. story_page.div.append(output_dom)
  137. story_dom.append(story_page)
  138. pi = pi+1
  139. # create main html file from filled template html dom
  140. book_html_f = os.path.join(_BUILD_d,'stories.html')
  141. with open(book_html_f, 'w') as fp:
  142. fp.write(template_dom.prettify(formatter=None))
  143. def hyphenate(node):
  144. # print("hyphenate")
  145. nodetext = node.get_text()
  146. # print(nodetext)
  147. nodestr = str(node)
  148. # print(nodestr)
  149. for word in nodetext.split(' '):
  150. # do not hyphenate if it's not a real word
  151. if len(word) < 5 or re.search('\w+', word) == None:
  152. continue
  153. # cleaning word
  154. # remove all non-alphanumerical characteres duplicated or more
  155. word = re.sub('\W{2,}', '', word)
  156. # remove all non-alphanumerical at the begining of word
  157. word = re.sub('^\W', '', word)
  158. # remove all non-alphanumerical at the end of word
  159. word = re.sub('\W$', '', word)
  160. # remove all word remaing having special chars
  161. if re.search('\W+', word):
  162. continue
  163. # hyphenate word
  164. word_hyphenated = _H_FR.inserted(word)
  165. # remove hyphen precedeted by less than 3 letters
  166. word_hyphenated = re.sub(r'^(\w{,2})-', r'\1', word_hyphenated)
  167. # remove hyphen followed by less than 3 letters
  168. word_hyphenated = re.sub(r'-(\w{,2})$', r'\1', word_hyphenated)
  169. # replace scores by html elemt &shy;
  170. word_hyphenated = re.sub(r'(\w)-(\w)', r'\1&shy;\2', word_hyphenated)
  171. # replace double scores by score+$shy;
  172. word_hyphenated = re.sub(r'--', r'-&shy;', word_hyphenated)
  173. # TODO: attention au date 1950-1960, le tiret disparait
  174. # print(word_hyphenated)
  175. if re.search('\b+', word):
  176. print(word+" | "+word_hyphenated)
  177. try:
  178. # replace word by hyhanated_word on source
  179. nodestr = re.sub(word, word_hyphenated, nodestr)
  180. # replaced_str_dom = BeautifulSoup(replaced_str, 'html.parser')
  181. # node.string = replaced_str
  182. # node.string.replace_with(node.string)
  183. except Exception as e:
  184. print(_ERROR_PREF+'Replacement error with \033[1m'+word+'\033[0m | \033[1m'+word_hyphenated+"\033[0m")
  185. print(e)
  186. print(node.string)
  187. print('[//]')
  188. pass
  189. # add none breaking spaces
  190. nbspzr_before = ['»', '\!', '\?', ':', ';']
  191. for char in nbspzr_before:
  192. nodestr = re.sub(r'(\w|>)\s('+char+')', r'\1&nbsp;\2', nodestr)
  193. nbspzr_after = ['«']
  194. for char in nbspzr_after:
  195. nodestr = re.sub(r'('+char+')\s(\w|<)', r'\1&nbsp;\2', nodestr)
  196. # print(nodestr)
  197. # replace node by hyphenated one
  198. node.replace_with(nodestr)
  199. if __name__ == "__main__":
  200. main()