Reworked download method use ORM instead of straight SQL. Hoping this fixes the database timeout issue. Also added requirements.txt with all the python packages needed
This commit is contained in:
Dan Dembinski
@@ -3,9 +3,9 @@ ADD main.py /
|
||||
|
||||
VOLUME /download/
|
||||
|
||||
RUN pip install -r requirements.txt
|
||||
RUN pip install manga-py
|
||||
RUN pip install sqlalchemy
|
||||
RUN pip install pymysql
|
||||
|
||||
|
||||
|
||||
CMD ["python", "./main.py"]
|
||||
162
main.py
162
main.py
@@ -1,4 +1,6 @@
|
||||
import sqlalchemy as db
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask import Flask
|
||||
|
||||
import json
|
||||
import time
|
||||
# from manga_py1.parser import Parser
|
||||
@@ -6,94 +8,92 @@ import time
|
||||
# from Downloader import my_awesome_handler
|
||||
import os
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
|
||||
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
class downloaderSettings(db.Model):
|
||||
ID = db.Column(db.Integer, primary_key=True)
|
||||
path = db.Column(db.Text)
|
||||
|
||||
|
||||
class manga(db.Model):
|
||||
manga_id = db.Column(db.Integer, primary_key=True)
|
||||
name = db.Column(db.Text)
|
||||
url = db.Column(db.Text)
|
||||
url_flag = db.Column(db.Text)
|
||||
view_format = db.Column(db.Integer)
|
||||
unknown_value = db.Column(db.Integer)
|
||||
active = db.Column(db.Boolean, default=1)
|
||||
|
||||
|
||||
class url_metadata(db.Model):
|
||||
url_id = db.Column(db.Integer, primary_key=True)
|
||||
source_name = db.Column(db.VARCHAR(length=10000))
|
||||
flag = db.Column(db.Text)
|
||||
base_url = db.Column(db.VARCHAR(length=10000))
|
||||
|
||||
|
||||
@app.route('/download', methods=['GET'])
|
||||
def download():
|
||||
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
|
||||
# db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
|
||||
|
||||
engine = db.create_engine(db_URI)
|
||||
con = engine.connect()
|
||||
|
||||
metadata = db.MetaData()
|
||||
|
||||
url_metadata = db.Table('url_metadata', metadata, autoload = True, autoload_with= engine)
|
||||
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
|
||||
# query = db.select([manga.columns.url, url_metadata.columns.base_url, manga.columns.url_flag])
|
||||
query = db.text(""" select manga.url, url_metadata.base_url , manga.name from manga join url_metadata on manga.url_flag = url_metadata.flag """)
|
||||
result = con.execute(query).fetchall()
|
||||
y=0
|
||||
url = []
|
||||
base = []
|
||||
name = []
|
||||
|
||||
# result = db.session.query(manga.base_url, manga.name, manga.url, url_metadata.base_url, url_metadata.flag).join(manga.url_flag == url_metadata.flag).all()
|
||||
result = db.session.query(manga, url_metadata).join(url_metadata, manga.url_flag==url_metadata.flag).all()
|
||||
|
||||
# PATH = 'Manga/'
|
||||
# PATH = 'smb://MediaServer._smb._tcp.local/Downloads/Manga/'
|
||||
PATH = '/download'
|
||||
for x in result:
|
||||
# print(result[y][0])
|
||||
url.append(result[y][0])
|
||||
base.append(result[y][1])
|
||||
name.append(result[y][2])
|
||||
|
||||
# print(base[y]+url[y])
|
||||
query = """ select active from manga where manga_id = """ + str(y+1)
|
||||
try:
|
||||
active = con.execute(query).fetchone()
|
||||
except engine.Error as e:
|
||||
if 'MySQL server has gone away' in str(e):
|
||||
con = engine.connect()
|
||||
for each in result:
|
||||
print("""manga-py """ + each.url_metadata.base_url+each.manga.url + """ -n \"""" + each.manga.name + """\" -d \"""" + PATH + """\" -z -N""")
|
||||
# os.system(execute)
|
||||
|
||||
if active[0] != 0:
|
||||
print("""manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N """)
|
||||
return "neat"
|
||||
|
||||
execute = """manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N"""
|
||||
os.system(execute)
|
||||
y = y+1
|
||||
|
||||
def upload_jason():
|
||||
url = []
|
||||
base = []
|
||||
name = []
|
||||
y = 0
|
||||
z = 0
|
||||
|
||||
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
|
||||
|
||||
engine = db.create_engine(db_URI)
|
||||
con = engine.connect()
|
||||
metadata = db.MetaData()
|
||||
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
|
||||
|
||||
with open("tach.json", "r") as tf:
|
||||
data = json.load(tf)
|
||||
for x in data["mangas"]:
|
||||
# Manga Name
|
||||
name.append(data["mangas"][y]["manga"][1])
|
||||
# Mange URL
|
||||
url.append(data["mangas"][y]["manga"][0])
|
||||
# Mange Base URL
|
||||
base.append(data["mangas"][y]["manga"][2])
|
||||
y=y+1
|
||||
# query = """ Delete from manga"""
|
||||
# def upload_jason():
|
||||
# url = []
|
||||
# base = []
|
||||
# name = []
|
||||
# y = 0
|
||||
# z = 0
|
||||
#
|
||||
# db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
|
||||
#
|
||||
# engine = db.create_engine(db_URI)
|
||||
# con = engine.connect()
|
||||
# metadata = db.MetaData()
|
||||
# manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
|
||||
#
|
||||
# with open("tach.json", "r") as tf:
|
||||
# data = json.load(tf)
|
||||
# for x in data["mangas"]:
|
||||
# # Manga Name
|
||||
# name.append(data["mangas"][y]["manga"][1])
|
||||
# # Mange URL
|
||||
# url.append(data["mangas"][y]["manga"][0])
|
||||
# # Mange Base URL
|
||||
# base.append(data["mangas"][y]["manga"][2])
|
||||
# y=y+1
|
||||
# # query = """ Delete from manga"""
|
||||
# # con.execute(query)
|
||||
# # query = """ Alter table manga auto_increment = 1 """
|
||||
# # con.execute(query)
|
||||
# for x in name:
|
||||
# query = """select name from manga where name = \"""" + name[z] + """\""""
|
||||
# exists = con.execute(query).fetchone()
|
||||
# if exists == None or exists[0] != name[z]:
|
||||
# query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
|
||||
# print(query)
|
||||
# con.execute(query)
|
||||
# query = """ Alter table manga auto_increment = 1 """
|
||||
# con.execute(query)
|
||||
for x in name:
|
||||
query = """select name from manga where name = \"""" + name[z] + """\""""
|
||||
exists = con.execute(query).fetchone()
|
||||
if exists == None or exists[0] != name[z]:
|
||||
query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
|
||||
print(query)
|
||||
con.execute(query)
|
||||
else:
|
||||
continue
|
||||
z=z+1
|
||||
# else:
|
||||
# continue
|
||||
# z=z+1
|
||||
|
||||
while True:
|
||||
print("1. Download \n 2. Upload\n 3. Exit")
|
||||
download()
|
||||
time.sleep(3600)
|
||||
# val = input(">>>")
|
||||
# if val == "1":
|
||||
# download()
|
||||
# elif val == "2":
|
||||
# upload_jason()
|
||||
# elif val == "3":
|
||||
# exit()
|
||||
if __name__ == "__main__":
|
||||
app.run(host='0.0.0.0', debug=True)
|
||||
|
||||
8
requirements.txt
Normal file
8
requirements.txt
Normal file
@@ -0,0 +1,8 @@
|
||||
click==7.1.1
|
||||
Flask==1.1.1
|
||||
Flask-SQLAlchemy==2.4.1
|
||||
itsdangerous==1.1.0
|
||||
Jinja2==2.11.1
|
||||
MarkupSafe==1.1.1
|
||||
PyMySQL==0.9.3
|
||||
Werkzeug==1.0.0
|
||||
Reference in New Issue
Block a user