162 lines
4.8 KiB
Python
162 lines
4.8 KiB
Python
from bs4 import BeautifulSoup
|
|
import requests
|
|
import zipfile
|
|
from pandas import read_csv, set_option
|
|
from sqlalchemy import create_engine, Column, Integer, String, Float
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|
from sqlalchemy.orm import sessionmaker
|
|
from os import remove
|
|
import logging
|
|
|
|
logging.basicConfig(filename='wspr.log', format='%(asctime)s %(message)s', level=logging.INFO)
|
|
|
|
callsign = 'KD8TUF'
|
|
linkArray = []
|
|
testing = False
|
|
|
|
set_option('display.max_rows', 3)
|
|
set_option('display.max_columns', None)
|
|
set_option('display.width', None)
|
|
set_option('display.max_colwidth', None)
|
|
|
|
engine = create_engine('sqlite:///wspr.db')
|
|
engine.connect()
|
|
Base = declarative_base()
|
|
|
|
|
|
class Receive(Base):
|
|
__tablename__ = 'Receive'
|
|
rowId = Column(Integer)
|
|
SpotID = Column(Integer, primary_key=True, autoincrement=True)
|
|
Timestamp = Column(String)
|
|
Reporter = Column(String)
|
|
reporterGrid = Column(String)
|
|
SNR = Column(Integer)
|
|
Freq = Column(Float)
|
|
CallSign = Column(String)
|
|
Power = Column(Integer)
|
|
Grid = Column(String)
|
|
Drift = Column(Integer)
|
|
Distance = Column(Integer)
|
|
Azimuth = Column(Integer)
|
|
Band = Column(Integer)
|
|
Version = Column(Float)
|
|
Code = Column(Integer)
|
|
|
|
|
|
class Sent(Base):
|
|
__tablename__ = 'Sent'
|
|
rowId = Column(Integer)
|
|
SpotID = Column(Integer, primary_key=True, autoincrement=True)
|
|
Timestamp = Column(String)
|
|
Reporter = Column(String)
|
|
reporterGrid = Column(String)
|
|
SNR = Column(Integer)
|
|
Freq = Column(Float)
|
|
CallSign = Column(String)
|
|
Power = Column(Integer)
|
|
Grid = Column(String)
|
|
Drift = Column(Integer)
|
|
Distance = Column(Integer)
|
|
Azimuth = Column(Integer)
|
|
Band = Column(Integer)
|
|
Version = Column(String)
|
|
Code = Column(Integer)
|
|
|
|
|
|
Session = sessionmaker(bind=engine)
|
|
session = Session()
|
|
|
|
# Original plan of scrapping olddb
|
|
# url = 'https://wsprnet.org/olddb'
|
|
# page = requests.get(url)
|
|
# soup = BeautifulSoup(page.content, 'html.parser')
|
|
# evenrow = soup.find_all(id='evenrow')
|
|
# for each in evenrow:
|
|
# optionvalue = each.find_all('option')
|
|
# for each in optionvalue:
|
|
# print(optionvalue[1])
|
|
# # print(evenrow)
|
|
|
|
if testing is False:
|
|
url = 'http://wsprnet.org/drupal/downloads'
|
|
page = requests.get(url)
|
|
soup = BeautifulSoup(page.content, 'html.parser')
|
|
results = soup.find_all("div", {"class": "field-item even"})
|
|
|
|
links = soup.find_all("a")
|
|
for link in links:
|
|
linkArray.append(link.get('href'))
|
|
ReportDownload = linkArray[-6:-5]
|
|
|
|
download = requests.get(ReportDownload[0], stream=True)
|
|
with open('./download.zip', 'wb') as fd:
|
|
for chunk in download.iter_content(chunk_size=128):
|
|
fd.write(chunk)
|
|
print(ReportDownload[0])
|
|
with zipfile.ZipFile('./download.zip', 'r') as zip:
|
|
zip.extractall('./')
|
|
filename = zip.namelist()
|
|
|
|
DailyData = read_csv(filename[0])
|
|
DailyData.columns = ['SpotID', 'Timestamp', 'Reporter', 'Reporter Grid', 'SNR', 'Freq', 'Call Sign', 'Grid', 'Power',
|
|
'Drift', 'Distance', 'Azimuth', 'Band', 'Version', 'Code']
|
|
|
|
meSee = DailyData[DailyData['Reporter'] == callsign]
|
|
ReceiveCount = 0
|
|
for index, row in meSee.iterrows():
|
|
if session.query(Receive.SpotID).filter_by(SpotID=row['SpotID']).first() is not None:
|
|
pass
|
|
else:
|
|
session.add(Receive(
|
|
SpotID=row['SpotID'],
|
|
Timestamp=row['Timestamp'],
|
|
Reporter=row['Reporter'],
|
|
reporterGrid=row['Reporter Grid'],
|
|
SNR=row['SNR'],
|
|
Freq=row['Freq'],
|
|
CallSign=row['Call Sign'],
|
|
Power=row['Power'],
|
|
Grid=row['Grid'],
|
|
Drift=row['Drift'],
|
|
Distance=row['Distance'],
|
|
Azimuth=row['Azimuth'],
|
|
Band=row['Band'],
|
|
Version=row['Version'],
|
|
Code=row['Code'])
|
|
)
|
|
ReceiveCount = ReceiveCount + 1
|
|
|
|
saw = DailyData[DailyData['Call Sign'] == callsign]
|
|
SentCount = 0
|
|
for index, row in saw.iterrows():
|
|
# print(row['SpotID'])
|
|
if session.query(Sent.SpotID).filter_by(SpotID=row['SpotID']).first() is not None:
|
|
pass
|
|
else:
|
|
session.add(Sent(
|
|
SpotID=row['SpotID'],
|
|
Timestamp=row['Timestamp'],
|
|
Reporter=row['Reporter'],
|
|
reporterGrid=row['Reporter Grid'],
|
|
SNR=row['SNR'],
|
|
Freq=row['Freq'],
|
|
CallSign=row['Call Sign'],
|
|
Power=row['Power'],
|
|
Grid=row['Grid'],
|
|
Drift=row['Drift'],
|
|
Distance=row['Distance'],
|
|
Azimuth=row['Azimuth'],
|
|
Band=row['Band'],
|
|
Version=row['Version'],
|
|
Code=row['Code'])
|
|
)
|
|
SentCount = SentCount+1
|
|
|
|
|
|
session.commit()
|
|
remove('./download.zip')
|
|
remove(filename[0])
|
|
|
|
logging.info('Import Completed. Imported %i Received and %i Sent', ReceiveCount, SentCount)
|