dan/mangaDownloader
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Reworked download method use ORM instead of straight SQL. Hoping this fixes the database timeout issue. Also added requirements.txt with all the python packages needed

Browse Source
This commit is contained in:
Dan Dembinski
2020-03-11 14:34:49 -04:00
parent abb101a8a7
commit 4581fce3b4
3 changed files with 93 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
main.py
View File

@@ -1,4 +1,6 @@
import sqlalchemy as db
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
import json
import time
# from manga_py1.parser import Parser
@@ -6,94 +8,92 @@ import time
# from Downloader import my_awesome_handler
import os
app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
db = SQLAlchemy(app)
class downloaderSettings(db.Model):
ID = db.Column(db.Integer, primary_key=True)
path = db.Column(db.Text)
class manga(db.Model):
manga_id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text)
url = db.Column(db.Text)
url_flag = db.Column(db.Text)
view_format = db.Column(db.Integer)
unknown_value = db.Column(db.Integer)
active = db.Column(db.Boolean, default=1)
class url_metadata(db.Model):
url_id = db.Column(db.Integer, primary_key=True)
source_name = db.Column(db.VARCHAR(length=10000))
flag = db.Column(db.Text)
base_url = db.Column(db.VARCHAR(length=10000))
@app.route('/download', methods=['GET'])
def download():
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
# db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
engine = db.create_engine(db_URI)
con = engine.connect()
metadata = db.MetaData()
url_metadata = db.Table('url_metadata', metadata, autoload = True, autoload_with= engine)
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
# query = db.select([manga.columns.url, url_metadata.columns.base_url, manga.columns.url_flag])
query = db.text(""" select manga.url, url_metadata.base_url , manga.name from manga join url_metadata on manga.url_flag = url_metadata.flag """)
result = con.execute(query).fetchall()
y=0
url = []
base = []
name = []
#PATH = 'Manga/'
# result = db.session.query(manga.base_url, manga.name, manga.url, url_metadata.base_url, url_metadata.flag).join(manga.url_flag == url_metadata.flag).all()
result = db.session.query(manga, url_metadata).join(url_metadata, manga.url_flag==url_metadata.flag).all()
# PATH = 'Manga/'
# PATH = 'smb://MediaServer._smb._tcp.local/Downloads/Manga/'
PATH = '/download'
for x in result:
# print(result[y][0])
url.append(result[y][0])
base.append(result[y][1])
name.append(result[y][2])
# print(base[y]+url[y])
query = """ select active from manga where manga_id = """ + str(y+1)
try:
active = con.execute(query).fetchone()
except engine.Error as e:
if 'MySQL server has gone away' in str(e):
con = engine.connect()
for each in result:
print("""manga-py """ + each.url_metadata.base_url+each.manga.url + """ -n \"""" + each.manga.name + """\" -d \"""" + PATH + """\" -z -N""")
# os.system(execute)
if active[0] != 0:
print("""manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N """)
return "neat"
execute = """manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N"""
os.system(execute)
y = y+1
# def upload_jason():
# url = []
# base = []
# name = []
# y = 0
# z = 0
#
# db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
#
# engine = db.create_engine(db_URI)
# con = engine.connect()
# metadata = db.MetaData()
# manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
#
# with open("tach.json", "r") as tf:
# data = json.load(tf)
# for x in data["mangas"]:
# # Manga Name
# name.append(data["mangas"][y]["manga"][1])
# # Mange URL
# url.append(data["mangas"][y]["manga"][0])
# # Mange Base URL
# base.append(data["mangas"][y]["manga"][2])
# y=y+1
# # query = """ Delete from manga"""
# # con.execute(query)
# # query = """ Alter table manga auto_increment = 1 """
# # con.execute(query)
# for x in name:
# query = """select name from manga where name = \"""" + name[z] + """\""""
# exists = con.execute(query).fetchone()
# if exists == None or exists[0] != name[z]:
# query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
# print(query)
# con.execute(query)
# else:
# continue
# z=z+1
def upload_jason():
url = []
base = []
name = []
y = 0
z = 0
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
engine = db.create_engine(db_URI)
con = engine.connect()
metadata = db.MetaData()
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
with open("tach.json", "r") as tf:
data = json.load(tf)
for x in data["mangas"]:
# Manga Name
name.append(data["mangas"][y]["manga"][1])
# Mange URL
url.append(data["mangas"][y]["manga"][0])
# Mange Base URL
base.append(data["mangas"][y]["manga"][2])
y=y+1
# query = """ Delete from manga"""
# con.execute(query)
# query = """ Alter table manga auto_increment = 1 """
# con.execute(query)
for x in name:
query = """select name from manga where name = \"""" + name[z] + """\""""
exists = con.execute(query).fetchone()
if exists == None or exists[0] != name[z]:
query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
print(query)
con.execute(query)
else:
continue
z=z+1
while True:
print("1. Download \n 2. Upload\n 3. Exit")
download()
time.sleep(3600)
# val = input(">>>")
# if val == "1":
# download()
# elif val == "2":
# upload_jason()
# elif val == "3":
# exit()
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=True)