From 4581fce3b45751d15ad7d177ba2b7774e36e3f0d Mon Sep 17 00:00:00 2001 From: Dan Dembinski Date: Wed, 11 Mar 2020 14:34:49 -0400 Subject: [PATCH] Reworked download method use ORM instead of straight SQL. Hoping this fixes the database timeout issue. Also added requirements.txt with all the python packages needed --- Dockerfile | 4 +- main.py | 166 +++++++++++++++++++++++------------------------ requirements.txt | 8 +++ 3 files changed, 93 insertions(+), 85 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index a0258d8..47dc959 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,9 @@ ADD main.py / VOLUME /download/ +RUN pip install -r requirements.txt RUN pip install manga-py -RUN pip install sqlalchemy -RUN pip install pymysql + CMD ["python", "./main.py"] \ No newline at end of file diff --git a/main.py b/main.py index 007ea09..14647dc 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,6 @@ -import sqlalchemy as db +from flask_sqlalchemy import SQLAlchemy +from flask import Flask + import json import time # from manga_py1.parser import Parser @@ -6,94 +8,92 @@ import time # from Downloader import my_awesome_handler import os +app = Flask(__name__) + +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False +app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' + +db = SQLAlchemy(app) + +class downloaderSettings(db.Model): + ID = db.Column(db.Integer, primary_key=True) + path = db.Column(db.Text) + + +class manga(db.Model): + manga_id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text) + url = db.Column(db.Text) + url_flag = db.Column(db.Text) + view_format = db.Column(db.Integer) + unknown_value = db.Column(db.Integer) + active = db.Column(db.Boolean, default=1) + + +class url_metadata(db.Model): + url_id = db.Column(db.Integer, primary_key=True) + source_name = db.Column(db.VARCHAR(length=10000)) + flag = db.Column(db.Text) + base_url = db.Column(db.VARCHAR(length=10000)) + + +@app.route('/download', methods=['GET']) def download(): - db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' + # db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' - engine = db.create_engine(db_URI) - con = engine.connect() - - metadata = db.MetaData() - - url_metadata = db.Table('url_metadata', metadata, autoload = True, autoload_with= engine) - manga = db.Table('manga', metadata, autoload=True, autoload_with=engine) # query = db.select([manga.columns.url, url_metadata.columns.base_url, manga.columns.url_flag]) - query = db.text(""" select manga.url, url_metadata.base_url , manga.name from manga join url_metadata on manga.url_flag = url_metadata.flag """) - result = con.execute(query).fetchall() - y=0 - url = [] - base = [] - name = [] - #PATH = 'Manga/' + + # result = db.session.query(manga.base_url, manga.name, manga.url, url_metadata.base_url, url_metadata.flag).join(manga.url_flag == url_metadata.flag).all() + result = db.session.query(manga, url_metadata).join(url_metadata, manga.url_flag==url_metadata.flag).all() + + # PATH = 'Manga/' # PATH = 'smb://MediaServer._smb._tcp.local/Downloads/Manga/' PATH = '/download' - for x in result: - # print(result[y][0]) - url.append(result[y][0]) - base.append(result[y][1]) - name.append(result[y][2]) - # print(base[y]+url[y]) - query = """ select active from manga where manga_id = """ + str(y+1) - try: - active = con.execute(query).fetchone() - except engine.Error as e: - if 'MySQL server has gone away' in str(e): - con = engine.connect() + for each in result: + print("""manga-py """ + each.url_metadata.base_url+each.manga.url + """ -n \"""" + each.manga.name + """\" -d \"""" + PATH + """\" -z -N""") + # os.system(execute) - if active[0] != 0: - print("""manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N """) + return "neat" - execute = """manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N""" - os.system(execute) - y = y+1 +# def upload_jason(): +# url = [] +# base = [] +# name = [] +# y = 0 +# z = 0 +# +# db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' +# +# engine = db.create_engine(db_URI) +# con = engine.connect() +# metadata = db.MetaData() +# manga = db.Table('manga', metadata, autoload=True, autoload_with=engine) +# +# with open("tach.json", "r") as tf: +# data = json.load(tf) +# for x in data["mangas"]: +# # Manga Name +# name.append(data["mangas"][y]["manga"][1]) +# # Mange URL +# url.append(data["mangas"][y]["manga"][0]) +# # Mange Base URL +# base.append(data["mangas"][y]["manga"][2]) +# y=y+1 +# # query = """ Delete from manga""" +# # con.execute(query) +# # query = """ Alter table manga auto_increment = 1 """ +# # con.execute(query) +# for x in name: +# query = """select name from manga where name = \"""" + name[z] + """\"""" +# exists = con.execute(query).fetchone() +# if exists == None or exists[0] != name[z]: +# query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")""" +# print(query) +# con.execute(query) +# else: +# continue +# z=z+1 -def upload_jason(): - url = [] - base = [] - name = [] - y = 0 - z = 0 - - db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' - - engine = db.create_engine(db_URI) - con = engine.connect() - metadata = db.MetaData() - manga = db.Table('manga', metadata, autoload=True, autoload_with=engine) - - with open("tach.json", "r") as tf: - data = json.load(tf) - for x in data["mangas"]: - # Manga Name - name.append(data["mangas"][y]["manga"][1]) - # Mange URL - url.append(data["mangas"][y]["manga"][0]) - # Mange Base URL - base.append(data["mangas"][y]["manga"][2]) - y=y+1 - # query = """ Delete from manga""" - # con.execute(query) - # query = """ Alter table manga auto_increment = 1 """ - # con.execute(query) - for x in name: - query = """select name from manga where name = \"""" + name[z] + """\"""" - exists = con.execute(query).fetchone() - if exists == None or exists[0] != name[z]: - query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")""" - print(query) - con.execute(query) - else: - continue - z=z+1 - -while True: - print("1. Download \n 2. Upload\n 3. Exit") - download() - time.sleep(3600) - # val = input(">>>") - # if val == "1": - # download() - # elif val == "2": - # upload_jason() - # elif val == "3": - # exit() \ No newline at end of file +if __name__ == "__main__": + app.run(host='0.0.0.0', debug=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..88ec1ca --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +click==7.1.1 +Flask==1.1.1 +Flask-SQLAlchemy==2.4.1 +itsdangerous==1.1.0 +Jinja2==2.11.1 +MarkupSafe==1.1.1 +PyMySQL==0.9.3 +Werkzeug==1.0.0