dan/mangaDownloader
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Reworked download method use ORM instead of straight SQL. Hoping this fixes the database timeout issue. Also added requirements.txt with all the python packages needed

Browse Source
This commit is contained in:
Dan Dembinski
2020-03-11 14:34:49 -04:00
parent abb101a8a7
commit 4581fce3b4
3 changed files with 93 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@@ -3,9 +3,9 @@ ADD main.py /
VOLUME /download/ VOLUME /download/
RUN pip install -r requirements.txt
RUN pip install manga-py RUN pip install manga-py
RUN pip install sqlalchemy
RUN pip install pymysql
CMD ["python", "./main.py"] CMD ["python", "./main.py"]

162
main.py
View File

@@ -1,4 +1,6 @@
import sqlalchemy as db from flask_sqlalchemy import SQLAlchemy
from flask import Flask
import json import json
import time import time
# from manga_py1.parser import Parser # from manga_py1.parser import Parser
@@ -6,94 +8,92 @@ import time
# from Downloader import my_awesome_handler # from Downloader import my_awesome_handler
import os import os
app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
db = SQLAlchemy(app)
class downloaderSettings(db.Model):
ID = db.Column(db.Integer, primary_key=True)
path = db.Column(db.Text)
class manga(db.Model):
manga_id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text)
url = db.Column(db.Text)
url_flag = db.Column(db.Text)
view_format = db.Column(db.Integer)
unknown_value = db.Column(db.Integer)
active = db.Column(db.Boolean, default=1)
class url_metadata(db.Model):
url_id = db.Column(db.Integer, primary_key=True)
source_name = db.Column(db.VARCHAR(length=10000))
flag = db.Column(db.Text)
base_url = db.Column(db.VARCHAR(length=10000))
@app.route('/download', methods=['GET'])
def download(): def download():
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' # db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
engine = db.create_engine(db_URI)
con = engine.connect()
metadata = db.MetaData()
url_metadata = db.Table('url_metadata', metadata, autoload = True, autoload_with= engine)
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
# query = db.select([manga.columns.url, url_metadata.columns.base_url, manga.columns.url_flag]) # query = db.select([manga.columns.url, url_metadata.columns.base_url, manga.columns.url_flag])
query = db.text(""" select manga.url, url_metadata.base_url , manga.name from manga join url_metadata on manga.url_flag = url_metadata.flag """)
result = con.execute(query).fetchall() # result = db.session.query(manga.base_url, manga.name, manga.url, url_metadata.base_url, url_metadata.flag).join(manga.url_flag == url_metadata.flag).all()
y=0 result = db.session.query(manga, url_metadata).join(url_metadata, manga.url_flag==url_metadata.flag).all()
url = []
base = []
name = []
# PATH = 'Manga/' # PATH = 'Manga/'
# PATH = 'smb://MediaServer._smb._tcp.local/Downloads/Manga/' # PATH = 'smb://MediaServer._smb._tcp.local/Downloads/Manga/'
PATH = '/download' PATH = '/download'
for x in result:
# print(result[y][0])
url.append(result[y][0])
base.append(result[y][1])
name.append(result[y][2])
# print(base[y]+url[y]) for each in result:
query = """ select active from manga where manga_id = """ + str(y+1) print("""manga-py """ + each.url_metadata.base_url+each.manga.url + """ -n \"""" + each.manga.name + """\" -d \"""" + PATH + """\" -z -N""")
try: # os.system(execute)
active = con.execute(query).fetchone()
except engine.Error as e:
if 'MySQL server has gone away' in str(e):
con = engine.connect()
if active[0] != 0: return "neat"
print("""manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N """)
execute = """manga-py """ + base[y]+url[y] + """ -n \"""" + name[y] + """\" -d \"""" + PATH + """\" -z -N""" # def upload_jason():
os.system(execute) # url = []
y = y+1 # base = []
# name = []
def upload_jason(): # y = 0
url = [] # z = 0
base = [] #
name = [] # db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads'
y = 0 #
z = 0 # engine = db.create_engine(db_URI)
# con = engine.connect()
db_URI = 'mysql+pymysql://mdownload:mdownload@192.168.86.198:3306/Downloads' # metadata = db.MetaData()
# manga = db.Table('manga', metadata, autoload=True, autoload_with=engine)
engine = db.create_engine(db_URI) #
con = engine.connect() # with open("tach.json", "r") as tf:
metadata = db.MetaData() # data = json.load(tf)
manga = db.Table('manga', metadata, autoload=True, autoload_with=engine) # for x in data["mangas"]:
# # Manga Name
with open("tach.json", "r") as tf: # name.append(data["mangas"][y]["manga"][1])
data = json.load(tf) # # Mange URL
for x in data["mangas"]: # url.append(data["mangas"][y]["manga"][0])
# Manga Name # # Mange Base URL
name.append(data["mangas"][y]["manga"][1]) # base.append(data["mangas"][y]["manga"][2])
# Mange URL # y=y+1
url.append(data["mangas"][y]["manga"][0]) # # query = """ Delete from manga"""
# Mange Base URL # # con.execute(query)
base.append(data["mangas"][y]["manga"][2]) # # query = """ Alter table manga auto_increment = 1 """
y=y+1 # # con.execute(query)
# query = """ Delete from manga""" # for x in name:
# query = """select name from manga where name = \"""" + name[z] + """\""""
# exists = con.execute(query).fetchone()
# if exists == None or exists[0] != name[z]:
# query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
# print(query)
# con.execute(query) # con.execute(query)
# query = """ Alter table manga auto_increment = 1 """ # else:
# con.execute(query) # continue
for x in name: # z=z+1
query = """select name from manga where name = \"""" + name[z] + """\""""
exists = con.execute(query).fetchone()
if exists == None or exists[0] != name[z]:
query = """ insert into manga(name, url, url_flag, view_format, unknown_value) values (\"""" + name[z] + """\",\"""" + url[z] + """\",\"""" + str(base[z]) + """\",\"0\",\"0\")"""
print(query)
con.execute(query)
else:
continue
z=z+1
while True: if __name__ == "__main__":
print("1. Download \n 2. Upload\n 3. Exit") app.run(host='0.0.0.0', debug=True)
download()
time.sleep(3600)
# val = input(">>>")
# if val == "1":
# download()
# elif val == "2":
# upload_jason()
# elif val == "3":
# exit()

8
requirements.txt Normal file
View File

@@ -0,0 +1,8 @@
click==7.1.1
Flask==1.1.1
Flask-SQLAlchemy==2.4.1
itsdangerous==1.1.0
Jinja2==2.11.1
MarkupSafe==1.1.1
PyMySQL==0.9.3
Werkzeug==1.0.0