ADD mariadb database import, no addresses yet
This commit is contained in:
@ -2,13 +2,13 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import mariadb
|
||||
import sys
|
||||
import time
|
||||
import pypdf
|
||||
#from PyPDF2 import PdfReader
|
||||
from pypdf import PdfReader
|
||||
import re # regular expression
|
||||
import pandas as pd
|
||||
import datetime
|
||||
|
||||
__version__ = '1.0.0'
|
||||
__website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html'
|
||||
@ -32,7 +32,8 @@ def call_parser():
|
||||
# parser.add_argument('-s', '--server' default=__website__, required=False)
|
||||
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__))
|
||||
parser.add_argument('-v', '--verbose', action='append_const', const = 1)
|
||||
parser.add_argument('-p', '--path', default='Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
|
||||
# Rufzeichenliste_AT_Stand_010624.pdf
|
||||
parser.add_argument('-p', '--path', default='afu/Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
|
||||
# parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet
|
||||
parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout')
|
||||
parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format "<IP-Address>:<Port> <User> <Passwd>" or defined in .config')
|
||||
@ -176,7 +177,7 @@ def call_split_name(fullname, call, verbose):
|
||||
case 'de' | 'el':
|
||||
name = fullname[3:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
|
||||
case 'van' | 'von' :
|
||||
@ -185,12 +186,12 @@ def call_split_name(fullname, call, verbose):
|
||||
if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg"
|
||||
name = fullname[8:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
case 'della' : # Ancient Italian noble family "della Rowere"
|
||||
name = fullname[6:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
|
||||
if len(name) > 1:
|
||||
@ -282,7 +283,7 @@ def fix_typo(call, fullname, verbose=1):
|
||||
fix_typo.lines = None
|
||||
fix_typo.spaces = []
|
||||
|
||||
def call_data_record(line, mod_date, verbose):
|
||||
def call_data_record(line, mod_date, verbose, cur):
|
||||
|
||||
# we have to split the record with a cost-intensive regular expression
|
||||
# record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100%
|
||||
@ -311,6 +312,8 @@ def call_data_record(line, mod_date, verbose):
|
||||
address = records[3]
|
||||
permit_class = records[4]
|
||||
fullname = fix_typo(call, fullname, verbose)
|
||||
firstname = ''
|
||||
surname = ''
|
||||
# If there is a clubstation
|
||||
if is_clubstation(call):
|
||||
# Name starting with only one quotation marks e.g. " -- remove that one:
|
||||
@ -322,6 +325,7 @@ def call_data_record(line, mod_date, verbose):
|
||||
if os.path.exists(path):
|
||||
fullname = replace_substring_with_line(path, fullname, verbose)
|
||||
gender = '*'
|
||||
firstname = fullname.strip()
|
||||
elif fullname[0] == '*':
|
||||
gender = '*'
|
||||
else: # Try to split the YL or OMs Name, guess the gender
|
||||
@ -334,8 +338,27 @@ def call_data_record(line, mod_date, verbose):
|
||||
|
||||
print(f'Location: {location}, Address: {address}, Permit: {permit_class}')
|
||||
|
||||
created = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
call_data_record.cnt += 1 # increment the User_id
|
||||
user_id = call_data_record.cnt
|
||||
# print(created)
|
||||
statement = "INSERT INTO `callbook_user`(`user_id`,`call`,`firstname`,`surname`,`gender`,`created`,`modified`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
|
||||
data = (user_id,call,firstname,surname,gender,created,created)
|
||||
|
||||
def call_analyse_pdf(file, verbose):
|
||||
try:
|
||||
# cur.execute(f"INSERT INTO `callbook_user` (`call`,`firstname`,`surname`,\
|
||||
# `created`,`created_by`,`modified`,`modified_by`,`active`)\
|
||||
# VALUES ('{call}','{firstname}','{surname}','{created}','0','{created}','0','{created}');")
|
||||
# cur.execute(f'INSERT INTO `callbook_user` (`call`) VALUES ("{call}");')
|
||||
cur.execute(statement, data)
|
||||
except mariadb.Error as e:
|
||||
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
|
||||
call_data_record.cnt = 0
|
||||
|
||||
def call_analyse_pdf(file, verbose, cur):
|
||||
|
||||
# Define a regular expression to match tables
|
||||
|
||||
@ -343,7 +366,7 @@ def call_analyse_pdf(file, verbose):
|
||||
meta = reader.metadata
|
||||
if verbose:
|
||||
print(verbose)
|
||||
print(' Pages:', len(reader.pages))
|
||||
print(' Pages:', len(reader.pages))
|
||||
# All of the following could be None!
|
||||
print(f' Author: {meta.author}')
|
||||
print(f' Creator: {meta.creator}')
|
||||
@ -361,18 +384,60 @@ def call_analyse_pdf(file, verbose):
|
||||
|
||||
if verbose >= 2:
|
||||
print(line)
|
||||
call_data_record(line, meta.modification_date,verbose)
|
||||
call_data_record(line, meta.modification_date,verbose, cur)
|
||||
|
||||
def exec_sql_file(cursor, sql_file):
|
||||
statement = ''
|
||||
try:
|
||||
for line in open(sql_file):
|
||||
if line.strip().startswith('--'): # ignore sql comment lines
|
||||
continue
|
||||
if line.strip().endswith(';'): # keep appending lines that don't end in ';'
|
||||
statement += line
|
||||
try:
|
||||
cursor.execute(statement)
|
||||
except mariadb.Error as e: # (OperationalError, ProgrammingError) as e:
|
||||
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
|
||||
statement = ''
|
||||
else:
|
||||
statement += line
|
||||
except FileNotFoundError:
|
||||
print(f'The file {path} was not found.')
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# call_description()
|
||||
args = call_parser()
|
||||
|
||||
try:
|
||||
filename = call_website(**vars(args))
|
||||
if args.verbose > 1:
|
||||
print(f'Filename: {filename}')
|
||||
call_analyse_pdf(filename,args.verbose)
|
||||
sys.exit(0)
|
||||
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
conn = mariadb.connect(
|
||||
user = 'om',
|
||||
password = 'oe3tkt',
|
||||
host='127.0.0.1',
|
||||
port=3306,
|
||||
database='callbook'
|
||||
)
|
||||
except mariadb.Error as e:
|
||||
print(f'Error connectiong to MariaDB platform: {e}')
|
||||
sys.exit(5)
|
||||
|
||||
print(datetime.datetime.now(datetime.UTC))
|
||||
# Get Cursor
|
||||
cur = conn.cursor()
|
||||
sql_file = '.sql_init'
|
||||
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), sql_file)
|
||||
exec_sql_file(cur, path)
|
||||
|
||||
call_analyse_pdf(filename,args.verbose,cur)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
sys.exit(0)
|
||||
|
||||
Reference in New Issue
Block a user