phuoc/html_cleanup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added view and erase log

Browse Source
This commit is contained in:
Phuoc
2023-11-29 17:30:37 +01:00
parent cd426b4bad
commit b0c1a0e858
22 changed files with 895 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
.gitignore vendored
View File

@@ -1,50 +1,22 @@
# These are some examples of commonly ignored file patterns. *.egg
# You should customize this list as applicable to your project. *.egg-info
# Learn more about .gitignore: *.pyc
# https://www.atlassian.com/git/tutorials/saving-changes/gitignore *$py.class
*~
# Node artifact files *.sqlite
node_modules/ .coverage
coverage.xml
build/
dist/ dist/
.tox/
# Compiled Java class files nosetests.xml
*.class env*/
tmp/
# Compiled Python bytecode Data.fs*
*.py[cod] *.sublime-project
*.sublime-workspace
# Log files .*.sw?
*.log .sw?
# Package files
*.jar
# Maven
target/
dist/
# JetBrains IDE
.idea/
# Unit test reports
TEST*.xml
# Generated by MacOS
.DS_Store .DS_Store
coverage
# Generated by Windows test
Thumbs.db
# Applications
*.app
*.exe
*.war
# Large media files
*.mp4
*.tiff
*.avi
*.flv
*.mov
*.wmv

11
cleanup_html/__init__.py
View File

@@ -1,11 +0,0 @@
from pyramid.config import Configurator
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
with Configurator(settings=settings) as config:
config.include('pyramid_jinja2')
config.include('.routes')
config.scan()
return config.make_wsgi_app()

3
cleanup_html/routes.py
View File

@@ -1,3 +0,0 @@
def includeme(config):
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_route('home', '/')

BIN
cleanup_html/static/pyramid-16x16.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.3 KiB

BIN
cleanup_html/static/pyramid.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 13 KiB

32
cleanup_html/static/theme.css
View File

@@ -1,32 +0,0 @@
@import url(//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700);
body {
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-weight: 300;
color: #1c1b1b;
background: #ffffff;
}
h1,
h2,
h3,
h4,
h5,
h6 {
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-weight: 300;
}
p {
font-weight: 300;
}
button, input, optgroup, select, textarea {
color: black;
}
.font-normal {
font-weight: 400;
}
.font-semi-bold {
font-weight: 600;
}
.font-bold {
font-weight: 700;
}

8
cleanup_html/templates/404.jinja2
View File

@@ -1,8 +0,0 @@
{% extends "layout.jinja2" %}
{% block content %}
<div class="content">
<h1><span class="font-semi-bold">Pyramid</span> <span class="smaller">Starter project</span></h1>
<p class="lead"><span class="font-semi-bold">404</span> Page Not Found</p>
</div>
{% endblock content %}

49
cleanup_html/templates/layout.jinja2
View File

@@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="{{request.locale_name}}">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="pyramid web application">
<meta name="author" content="Pylons Project">
<link rel="shortcut icon" href="{{request.static_url('cleanup_html:static/pyramid-16x16.png')}}">
<title>Cookiecutter Starter project for the Pyramid Web Framework</title>
<!-- Bootstrap core CSS -->
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<!-- Custom styles for this scaffold -->
<link href="{{request.static_url('cleanup_html:static/theme.css')}}" rel="stylesheet">
<!-- HTML5 shiv and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
<script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js" integrity="sha384-0s5Pv64cNZJieYFkXYOTId2HMA2Lfb6q2nAcx2n0RTLUnCAoTTsS0nKEO27XyKcY" crossorigin="anonymous"></script>
<script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js" integrity="sha384-ZoaMbDF+4LeFxg6WdScQ9nnR1QC2MIRxA1O9KWEXQwns1G8UNyIEZIQidzb0T1fo" crossorigin="anonymous"></script>
<![endif]-->
</head>
<body>
<div class="container">
<div class="row">
{% block content %}
<p>No content</p>
{% endblock content %}
</div>
<div class="row">
<div class="copyright">
Copyright &copy; Pylons Project
</div>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="//code.jquery.com/jquery-1.12.4.min.js" integrity="sha256-ZosEbRLbNQzLpnKIkEdrPv7lOy9C27hHQ+Xp8a4MxAQ=" crossorigin="anonymous"></script>
<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
</body>
</html>

8
cleanup_html/templates/mytemplate.jinja2
View File

@@ -1,8 +0,0 @@
{% extends "layout.jinja2" %}
{% block content %}
<div class="content">
<h1><span class="font-semi-bold">Pyramid</span> <span class="smaller">Starter project</span></h1>
<p class="lead">Welcome to <span class="font-normal">{{project}}</span>, a&nbsp;Pyramid application generated&nbsp;by<br><span class="font-normal">Cookiecutter</span>.</p>
</div>
{% endblock content %}

0
cleanup_html/views/__init__.py
View File

6
cleanup_html/views/default.py
View File

@@ -1,6 +0,0 @@
from pyramid.view import view_config
@view_config(route_name='home', renderer='cleanup_html:templates/mytemplate.jinja2')
def my_view(request):
return {'project': 'cleanup_html'}

7
cleanup_html/views/notfound.py
View File

@@ -1,7 +0,0 @@
from pyramid.view import notfound_view_config
@notfound_view_config(renderer='cleanup_html:templates/404.jinja2')
def notfound_view(request):
request.response.status = 404
return {}

1
html_cleanup/routes.py
View File

@@ -1,3 +1,4 @@
def includeme(config): def includeme(config):
config.add_static_view('static', 'static', cache_max_age=3600) config.add_static_view('static', 'static', cache_max_age=3600)
config.add_route('home', '/') config.add_route('home', '/')
config.add_route('view_log', '/view_log')

5
html_cleanup/static/temp/clean_all_gender.png Normal file
View File

@@ -0,0 +1,5 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>untitled</title>
</head>

769
html_cleanup/static/temp/clean_clean_Nantes_anthologie_cinema_viet.htm Normal file
View File

@@ -0,0 +1,769 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>untitled</title>
</head>
<body>
<div class=WordSection1>
<h1><b>
Festival des 3 Continents, Nantes 24.11 - 3.12 2023</b></h1>
<h2><b>
Anthologie du cinéma vietnamien</b></h2>
<p align="justify">&nbsp;</p>
<h2>19 films vietnamiens
de 1974 à 2022</h2>
<p align="justify">&nbsp;</p>
<img src="yda_anthologie_cinema_viet_fig_3.jpg" alt="We will meet again" width="510" heigth="500"><br><br>
<p align="justify"><b>We will meet
again</b> / Đến hẹn lại lên - Tran Vu, 1974, 108’</p>
<p align="justify">Dimanche 26.11,
18:30 - Vendredi 1.12, 13:45</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Premier amour</b>
/ Mối tình đầu - Hai Ninh, 1977, 112’</p>
<p align="justify">Samedi 25.11, 13:45
- Jeudi 30.11, 20:30</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>The faces of may</b>
/ Tháng năm, những gương mặt- Dang Nhat Minh, 1975, 37’</p>
<img src="yda_anthologie_cinema_viet_fig_2.jpg" alt="Nostalgie de la campagne" width="510" heigth="500"><br><br>
<p align="justify"><b>Nostalgie de la
campagne</b>&nbsp;/ Thương nhớ đồng quê - Dang Nhat Minh,
1995, 116’</p>
<p align="justify">Mercredi 29.11,
20:30 - Vendredi 1.12, 17:00</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Chom et Sa</b>
/ Chom và Sa - Pham Ky Nam, 1979, 70’</p>
<p align="justify">Dimanche 26.11,
10:00 - Lundi 27.11, 16:15</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Hanoi through
whose eyes&nbsp;?</b> / Hà Nội trong mắt ai - Tran Van
Thuy, 1982, 45’</p>
<p align="justify"><b>The story of kindness
</b>or<b> How to behave</b>&nbsp;/ Chuyện tử tế - Tran Van
Thuy, 1987, 43’</p>
<p align="justify">Mardi 28.11, 16:00 -
Dimanche 3.12, 10:30</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Brothers</b>
/ Anh và em - Nguyen Huu Luyen &amp; Tran Vu, 1986, 87’</p>
<p align="justify">Dimanche 26.11, 20:45
- Jeudi 30.11, 18:30</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>La fille du
fleuve</b> / Cô gái trên sông - Dang Nhat Minh, 1987, 100’</p>
<p align="justify">Samedi 25.11, 20:45
- Jeudi 30.11, 10:15</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Troupe de cirque
ambulant</b> / Gánh xiếc rong - Viet Linh, 1988, 80’</p>
<p align="justify">Lundi 27.11, 17:30 -
Dimanche 3.12, 17:30</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Fairytale for a
17-year-old girl</b> /  Chuyện cổ tích cho tuổi 17 -
Nguyen Xuan Son, 1988, 77’</p>
<p align="justify">Mercredi 29.11,
13&nbsp;:00 - Samedi 2.12, 13&nbsp;:00</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>La lampe dans le
rêve</b> / Ngọn đèn trong mơ - Do Minh Tuan, 1988, 75’</p>
<p align="justify">Mardi 28.11, 10:15 -
Vendredi 1.12, 18:45</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Money,
Money&nbsp;!</b> / Tiền ơi&nbsp;! - Tran Vu &amp; Nguyen
Huu Luyen, 1989, 93’</p>
<p align="justify">Dimanche 26.11,
15:30 - Samedi 2.12, 15:45</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Enfance orageuse</b>
/ Tuổi thơ dữ dội - Nguyen Vinh Son, 1990, 135’</p>
<p align="justify">Lundi 27.11, 13:30 -
Samedi 2.12, 20:30</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>PLease forgive me</b>
/ Hãy tha thứ cho em - Luu Trong Ninh, 1992, 90’</p>
<p align="justify">Lundi 27.11, 18:30 -
Mercredi 29.11, 18:15</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>Piège d’amour</b>
/ Cạm bảy tình yêu - Pham Loc, 1992, 82’</p>
<p align="justify">Lundi 27.11, 10:30 -
Mercredi 29.11, 14:45</p>
<p align="justify">&nbsp;</p>
<p align="justify"><b>In the lane</b>
/ Ngõ hẹp - Bach Diep, 1993, 98’</p>
<p align="justify">Samedi 25.11, 10:15
- Mardi 28.11, 14:00</p>
<p align="justify">&nbsp;</p>
<img src="yda_anthologie_cinema_viet_fig_4.jpg" alt="L’Immeuble" width="510" heigth="500"><br><br>
<p align="justify"><b>L’Immeuble</b>
/ Chung cư - Viet Linh, 1999, 90’</p>
<p align="justify">Samedi 25.11, 16:15
- Mercredi 29.11, 15:00</p>
<p align="justify">&nbsp;</p>
<img src="yda_anthologie_cinema_viet_fig_1.jpg" alt="Dust & metal" width="510" heigth="510"><br><br>
<p align="justify"><b>Dust &amp; metal</b>
/ Cát bụi và kim loại - Esther Johnson, 2022, 83’</p>
<p align="justify">Vendredi 1.12, 20:30
- Dimanche 3.12, 18:00</p>
<p align="justify">&nbsp;</p>
<p align="justify">Anthologie du cinéma vietnamien&nbsp;:</p>
<p align="justify">
https://www.3continents.com/fr/programme/2023/anthologie-du-cinema-vietnamien/</a></p>
<p align="justify">Programme 2023, horaires et lieux&nbsp;:</p>
<p align="justify">
https://www.3continents.com/wp-content/uploads/f3c-prog-2023-40p-net-planche.pdf</a></p>
<p align="justify">&nbsp;</p>
<p align="justify">&nbsp;</p>
<p align="justify">&nbsp;</p>
<p align="justify">&nbsp;</p>
</div>
<div id="pDebug">
<div id="pDebugToolbarHandle">
<a title="Show Toolbar" id="pShowToolBarButton"
>
&#171;</a>
</div>
</div>
</body>
</html>

5
html_cleanup/static/temp/clean_proposed_file_name.html Normal file
View File

@@ -0,0 +1,5 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>untitled</title>
</head>

5
html_cleanup/static/temp/errors_log
View File

@@ -1,3 +1,2 @@
2023-11-29 11:40:58 - DSC03776.JPGERREUR: La taille du fichier dépasse la limite autorisée. Téléchargement refusé. 2023-11-29 17:22:16 [ Nantes_anthologie_cinema_viet.htm ] *** Cleanup réussi. ***
2023-11-29 11:44:29 - DSC03777.JPG - ERREUR: La taille du fichier dépasse la limite autorisée. Téléchargement refusé. 2023-11-29 17:23:56 [ Screenshot 2023-11-20 at 12.07.54.png ] Le format du fichier n'est pas valide. Téléchargement refusé.
2023-11-29 11:45:44 - devis prothèse dentaire CKT.pdf - ERREUR: Le format du fichier n'est pas valide. Téléchargement refusé.

13
html_cleanup/templates/home.jinja2
View File

@@ -8,6 +8,13 @@
</div> </div>
{% endif %} {% endif %}
<div class="well">
<ul>
<li>Seuls les fichiers au format <b>HTML</b> seront acceptés.</li>
<li>La taille du fichier ne doit pas <b>dépasser 10 Mo</b>.</li>
</ul>
</div>
<form id="uploadfile-form" action="{{ request.route_url('home') }}" method="post" <form id="uploadfile-form" action="{{ request.route_url('home') }}" method="post"
accept-charset="utf-8" enctype="multipart/form-data"> accept-charset="utf-8" enctype="multipart/form-data">
<div class="form-group"> <div class="form-group">
@@ -21,12 +28,6 @@
</div> </div>
</form> </form>
<div class="panel-footer">
<ul>
<li>Seuls les fichiers au format <b>HTML</b> seront acceptés.</li>
<li>La taille du fichier ne doit pas <b>dépasser 10 Mo</b>.</li>
</ul>
</div>
{% if file_url %} {% if file_url %}
<h2 class="text-info font-semi-bold">Cleanup réussi</h2> <h2 class="text-info font-semi-bold">Cleanup réussi</h2>

2
html_cleanup/templates/layout.jinja2
View File

@@ -8,7 +8,7 @@
<meta name="author" content="Pylons Project"> <meta name="author" content="Pylons Project">
<link rel="shortcut icon" href="{{request.static_url('html_cleanup:static/pyramid-16x16.png')}}"> <link rel="shortcut icon" href="{{request.static_url('html_cleanup:static/pyramid-16x16.png')}}">
<title>Cookiecutter Starter project for the Pyramid Web Framework</title> <title>HTML cleanup</title>
<!-- Bootstrap core CSS --> <!-- Bootstrap core CSS -->
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous"> <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">

19
html_cleanup/templates/view_log.jinja2 Normal file
View File

@@ -0,0 +1,19 @@
{% extends "layout.jinja2" %}
{% block content %}
<div class="well">
{% for line in lines %}
<div>{{ line }}</div>
{% endfor %}
</div>
{%if lines %}
<form id="view_log-form" action="{{ request.route_url('view_log') }}" method="post">
<div class="form-group">
<button id="eraseButton" class="btn btn-link" type="submit" name="form.submitted">[ Xóa log ]</button>
</div>
</form>
{% endif %}
{% endblock content %}

100
html_cleanup/views/default.py
View File

@@ -30,6 +30,43 @@ def home(request):
'file_name': file_name, 'file_name': file_name,
} }
def process_file(request, input_file, input_name):
# Check file mime type and size
# and if OK, process file
message = ''
ext_allowed = ['text/html']
max_size = 5 * (1024 ** 2) # 10 Mb
temp_folder = request.registry.settings['temp_folder']
logfile_name = os.path.join(temp_folder, 'errors_log')
mime = magic.from_buffer(input_file.read(), mime=True)
# types de fichiers autorisés ?
if mime not in ext_allowed:
message = "Le format du fichier n'est pas valide. Téléchargement refusé."
add_error2log(logfile_name, input_name, message)
else:
# lire la taille du fichier
input_file.seek(0, 2) #seek to end
filesize = input_file.tell()
input_file.seek(0) # back to original position
# controler la taille du fichier
if filesize > max_size:
message = "La taille du fichier dépasse la limite autorisée. Téléchargement refusé."
add_error2log(logfile_name, input_name, message)
else:
# controle OK, traiter le fichier
output_name = "clean_" + input_name
output_file = os.path.join(temp_folder, output_name)
message = clean_html(input_file, output_file)
if message:
add_error2log(logfile_name, input_name, message)
else:
add_error2log(logfile_name, input_name, "*** Cleanup réussi. ***")
return message
def clean_html(input_file, output_file): def clean_html(input_file, output_file):
# cleanup undesirable tags in html file # cleanup undesirable tags in html file
encoding = "utf-8" encoding = "utf-8"
@@ -122,46 +159,37 @@ def clean_html(input_file, output_file):
return message return message
def process_file(request, input_file, input_name):
# Check file mime type and size
# and if OK, process file
message = ''
ext_allowed = ['text/html', 'image/png', 'image/jpeg']
max_size = 5 * (1024 ** 2) # 10 Mb
temp_folder = request.registry.settings['temp_folder']
logfile_name = os.path.join(temp_folder, 'errors_log')
mime = magic.from_buffer(input_file.read(), mime=True)
# types de fichiers autorisés ?
if mime not in ext_allowed:
message = "Le format du fichier n'est pas valide. Téléchargement refusé."
add_error2log(logfile_name, input_name, message)
else:
# lire la taille du fichier
input_file.seek(0, 2) #seek to end
filesize = input_file.tell()
input_file.seek(0) # back to original position
# controler la taille du fichier
if filesize > max_size:
message = "La taille du fichier dépasse la limite autorisée. Téléchargement refusé."
add_error2log(logfile_name, input_name, message)
else:
# controle OK, traiter le fichier
output_name = "clean_" + input_name
output_file = os.path.join(temp_folder, output_name)
message = clean_html(input_file, output_file)
if message:
add_error2log(logfile_name, input_name, message)
return message
def add_error2log(logfile_name, input_name, message): def add_error2log(logfile_name, input_name, message):
# ecrire message d'erreur dans le log # ecrire message d'erreur dans le log
# Get the current date and time # Get the current date and time
now = datetime.datetime.now() now = datetime.datetime.now()
with open(logfile_name, 'a', encoding='utf-8') as file: with open(logfile_name, 'a', encoding='utf-8') as file:
line = now.strftime("%Y-%m-%d %H:%M:%S") + ' [' + input_name + '] ' + f'{message}\n' line = now.strftime("%Y-%m-%d %H:%M:%S") + ' [ ' + input_name + ' ] ' + f'{message}\n'
file.write(line) file.write(line)
@view_config(route_name='view_log', renderer='html_cleanup:templates/view_log.jinja2')
def view_log(request):
message = ''
temp_folder = request.registry.settings['temp_folder']
logfile_name = os.path.join(temp_folder, 'errors_log')
lines = []
if 'form.submitted' in request.params:
if os.path.exists(logfile_name):
os.remove(logfile_name)
# Ouvrir le fichier en lecture seule
try:
file = open(logfile_name, "r")
lines = file.readlines()
file.close()
except Exception as e:
pass
return {
'page_title': "HTML cleanup log",
'message': message,
'lines': lines,
}

2
setup.py
View File

@@ -25,7 +25,7 @@ tests_require = [
setup( setup(
name='html_cleanup', name='html_cleanup',
version='0.0', version='1.0',
description='html_cleanup', description='html_cleanup',
long_description=README + '\n\n' + CHANGES, long_description=README + '\n\n' + CHANGES,
classifiers=[ classifiers=[