public/scripts
public/scripts
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ADD mariadb database import, no addresses yet

Browse Source
This commit is contained in:
Thomas Kuschel 2024-06-19 19:51:08 +02:00
parent 5682220486
commit 438cccc474
3 changed files with 278 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

324
afu/.gender
View File

@ -364,165 +364,165 @@ m Yannic
m Zeljko m Zeljko
m Zlatko m Zlatko
m Zvonko m Zvonko
w Adelheid f Adelheid
w Alexandra f Alexandra
w Andrea f Andrea
w Angela f Angela
w Angelika f Angelika
w Anita f Anita
w Anna f Anna
w Anna-Maria f Anna-Maria
w Anneliese f Anneliese
w Annemarie f Annemarie
w Astrid f Astrid
w Auguste f Auguste
w Barbara f Barbara
w Beatrice f Beatrice
w Beatrix f Beatrix
w Bernadette f Bernadette
w Bernardine f Bernardine
w Bettina f Bettina
w Bianca f Bianca
w Birgit f Birgit
w Brigitte f Brigitte
w Britta f Britta
w Carmen f Carmen
w Chiara f Chiara
w Christa f Christa
w Christine f Christine
w Christl f Christl
w Cornelia f Cornelia
w Durdica f Durdica
w Edeltraud f Edeltraud
w Elfriede f Elfriede
w Elisabeth f Elisabeth
w Elke f Elke
w Erdmuthe f Erdmuthe
w Ernestine f Ernestine
w Eva-Maria f Eva-Maria
w Eveline f Eveline
w Evelyn f Evelyn
w Flora f Flora
w Franziska f Franziska
w Frederike f Frederike
w Frieda f Frieda
w Friederike f Friederike
w Gabriela f Gabriela
w Gabriella f Gabriella
w Gabriele f Gabriele
w Gerda f Gerda
w Gerlinde f Gerlinde
w Gertraude f Gertraude
w Gertrude f Gertrude
w Gisela f Gisela
w Gudrun f Gudrun
w Gunhild f Gunhild
w Gustav f Gustav
w Hannelore f Hannelore
w Heidelinde f Heidelinde
w Heidi f Heidi
w Heidrun f Heidrun
w Helga f Helga
w Hemma f Hemma
w Hermine f Hermine
w Herta f Herta
w Hildegard f Hildegard
w Ingeborg f Ingeborg
w Ingeburg f Ingeburg
w Ingrid f Ingrid
w Isabel f Isabel
w Isabella f Isabella
w Isolde f Isolde
w Janet f Janet
w Jasmin f Jasmin
w Jemilla-Katalin f Jemilla-Katalin
w Jessica f Jessica
w Johanna f Johanna
w Josefine f Josefine
w Julia f Julia
w Juliana f Juliana
w Jutta f Jutta
w Karin f Karin
w Karolina f Karolina
w Karoline f Karoline
w Katharina f Katharina
w Kathrin f Kathrin
w Katja f Katja
w Katrin f Katrin
w Kerstin f Kerstin
w Klaudia f Klaudia
w Laila f Laila
w Larissa f Larissa
w Leonie f Leonie
w Lieselotte f Lieselotte
w Ligia f Ligia
w Lisbeth f Lisbeth
w Lygia f Lygia
w Luisa f Luisa
w Luiza f Luiza
w Magdalena f Magdalena
w Manfreda f Manfreda
w Manuela f Manuela
w Margareta f Margareta
w Margarethe f Margarethe
w Margot f Margot
w Margret f Margret
w Marianne f Marianne
w Marie-Luise f Marie-Luise
w Marina f Marina
w Marion f Marion
w Marlene f Marlene
w Martha f Martha
w Martina f Martina
w Mathilde f Mathilde
w Mechthild f Mechthild
w Melanie f Melanie
w Michaela f Michaela
w Nadine f Nadine
w Natasa f Natasa
w Natascha f Natascha
w Nicole f Nicole
w Nikolitsa f Nikolitsa
w Nina f Nina
w Noriko f Noriko
w Olivia f Olivia
w Patrizia f Patrizia
w Paulina f Paulina
w Pauline f Pauline
w Phaedra f Phaedra
w Regina f Regina
w Reinhilde f Reinhilde
w Renate f Renate
w Renee f Renee
w Rosina f Rosina
w Roswitha f Roswitha
w Sabine f Sabine
w Sandra f Sandra
w Senada f Senada
w Ricarda f Ricarda
w Sieglinde f Sieglinde
w Silvia f Silvia
w Simone f Simone
w Solveig f Solveig
w Sonja f Sonja
w Sophia f Sophia
w Sophie f Sophie
w Stefanie f Stefanie
w Steffi f Steffi
w Stephanie f Stephanie
w Susanne f Susanne
w Sybille f Sybille
w Tadeja f Tadeja
w Tamara f Tamara
w Tanja f Tanja
w Tatjana f Tatjana
w Theresia f Theresia
w Ulrike f Ulrike
w Ursula f Ursula
w Valerie f Valerie
w Valery f Valery
w Veronika f Veronika
w Victoria f Victoria
w Waldtraud f Waldtraud
w Waltraud f Waltraud
w Yvonne f Yvonne

37
afu/.sql_init Normal file
View File

@ -0,0 +1,37 @@
-- Initialize the tables for the callbook program
CREATE TABLE IF NOT EXISTS `callbook_user`(
`id` SERIAL,
`user_id` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`call` VARCHAR(10) NOT NULL DEFAULT '' COMMENT 'amateur radio call sign e.g. OE8ABC',
`firstname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'forename, given name',
`surname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'family name, last name',
`gender` CHAR(1) NOT NULL DEFAULT '' COMMENT 'gender guessed from the first name',
`created` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`created_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`modified` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`modified_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`active` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`inactive` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`address_id` INT(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'link to id of callbook_adress',
PRIMARY KEY(`id`)
) AUTO_INCREMENT=0
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `callbook_address`(
`id` SERIAL,
`location` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original location from the import',
`address` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original address from the import',
`postal_code` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'also known as Zip code, "Postleitzahl"',
`org_city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
`city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
`federal_state_code` TINYINT NOT NULL DEFAULT -1
COMMENT 'Austria: 0...outside, 1...Vienna, 2...Salzburg, 3...Lower Austria, 4...Burgenland, 5...Upper Austria, 6...Styria, 7...Tyrol, 8...Carinthia, 9...Vorarlberg',
`lat` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'latitude',
`long` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'longitude',
`qth` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'Maidenhead locator, QTH locator',
PRIMARY KEY(`id`)
) AUTO_INCREMENT=0
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_unicode_ci;

93
afu/callbook.py
View File

@ -2,13 +2,13 @@
import argparse import argparse
import os import os
import mariadb
import sys import sys
import time import time
import pypdf import pypdf
#from PyPDF2 import PdfReader
from pypdf import PdfReader from pypdf import PdfReader
import re # regular expression import re # regular expression
import pandas as pd import datetime
__version__ = '1.0.0' __version__ = '1.0.0'
__website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html' __website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html'
@ -32,7 +32,8 @@ def call_parser():
# parser.add_argument('-s', '--server' default=__website__, required=False) # parser.add_argument('-s', '--server' default=__website__, required=False)
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__)) parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__))
parser.add_argument('-v', '--verbose', action='append_const', const = 1) parser.add_argument('-v', '--verbose', action='append_const', const = 1)
parser.add_argument('-p', '--path', default='Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists') # Rufzeichenliste_AT_Stand_010624.pdf
parser.add_argument('-p', '--path', default='afu/Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
# parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet # parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet
parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout') parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout')
parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format "<IP-Address>:<Port> <User> <Passwd>" or defined in .config') parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format "<IP-Address>:<Port> <User> <Passwd>" or defined in .config')
@ -176,7 +177,7 @@ def call_split_name(fullname, call, verbose):
case 'de' | 'el': case 'de' | 'el':
name = fullname[3:].split(' ',1) name = fullname[3:].split(' ',1)
surname = surname.lower() + ' ' + name[0] surname = surname.lower() + ' ' + name[0]
if verbose > 0: if verbose > 1:
print(f'## {fullname} --> {surname} ##') print(f'## {fullname} --> {surname} ##')
case 'van' | 'von' : case 'van' | 'von' :
@ -185,12 +186,12 @@ def call_split_name(fullname, call, verbose):
if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg" if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg"
name = fullname[8:].split(' ',1) name = fullname[8:].split(' ',1)
surname = surname.lower() + ' ' + name[0] surname = surname.lower() + ' ' + name[0]
if verbose > 0: if verbose > 1:
print(f'## {fullname} --> {surname} ##') print(f'## {fullname} --> {surname} ##')
case 'della' : # Ancient Italian noble family "della Rowere" case 'della' : # Ancient Italian noble family "della Rowere"
name = fullname[6:].split(' ',1) name = fullname[6:].split(' ',1)
surname = surname.lower() + ' ' + name[0] surname = surname.lower() + ' ' + name[0]
if verbose > 0: if verbose > 1:
print(f'## {fullname} --> {surname} ##') print(f'## {fullname} --> {surname} ##')
if len(name) > 1: if len(name) > 1:
@ -282,7 +283,7 @@ def fix_typo(call, fullname, verbose=1):
fix_typo.lines = None fix_typo.lines = None
fix_typo.spaces = [] fix_typo.spaces = []
def call_data_record(line, mod_date, verbose): def call_data_record(line, mod_date, verbose, cur):
# we have to split the record with a cost-intensive regular expression # we have to split the record with a cost-intensive regular expression
# record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100% # record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100%
@ -311,6 +312,8 @@ def call_data_record(line, mod_date, verbose):
address = records[3] address = records[3]
permit_class = records[4] permit_class = records[4]
fullname = fix_typo(call, fullname, verbose) fullname = fix_typo(call, fullname, verbose)
firstname = ''
surname = ''
# If there is a clubstation # If there is a clubstation
if is_clubstation(call): if is_clubstation(call):
# Name starting with only one quotation marks e.g. " -- remove that one: # Name starting with only one quotation marks e.g. " -- remove that one:
@ -322,6 +325,7 @@ def call_data_record(line, mod_date, verbose):
if os.path.exists(path): if os.path.exists(path):
fullname = replace_substring_with_line(path, fullname, verbose) fullname = replace_substring_with_line(path, fullname, verbose)
gender = '*' gender = '*'
firstname = fullname.strip()
elif fullname[0] == '*': elif fullname[0] == '*':
gender = '*' gender = '*'
else: # Try to split the YL or OMs Name, guess the gender else: # Try to split the YL or OMs Name, guess the gender
@ -334,8 +338,27 @@ def call_data_record(line, mod_date, verbose):
print(f'Location: {location}, Address: {address}, Permit: {permit_class}') print(f'Location: {location}, Address: {address}, Permit: {permit_class}')
created = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
call_data_record.cnt += 1 # increment the User_id
user_id = call_data_record.cnt
# print(created)
statement = "INSERT INTO `callbook_user`(`user_id`,`call`,`firstname`,`surname`,`gender`,`created`,`modified`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
data = (user_id,call,firstname,surname,gender,created,created)
def call_analyse_pdf(file, verbose): try:
# cur.execute(f"INSERT INTO `callbook_user` (`call`,`firstname`,`surname`,\
# `created`,`created_by`,`modified`,`modified_by`,`active`)\
# VALUES ('{call}','{firstname}','{surname}','{created}','0','{created}','0','{created}');")
# cur.execute(f'INSERT INTO `callbook_user` (`call`) VALUES ("{call}");')
cur.execute(statement, data)
except mariadb.Error as e:
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
except Exception as e:
print('Error: {}'.format(e), file=sys.stderr)
call_data_record.cnt = 0
def call_analyse_pdf(file, verbose, cur):
# Define a regular expression to match tables # Define a regular expression to match tables
@ -343,7 +366,7 @@ def call_analyse_pdf(file, verbose):
meta = reader.metadata meta = reader.metadata
if verbose: if verbose:
print(verbose) print(verbose)
print(' Pages:', len(reader.pages)) print(' Pages:', len(reader.pages))
# All of the following could be None! # All of the following could be None!
print(f' Author: {meta.author}') print(f' Author: {meta.author}')
print(f' Creator: {meta.creator}') print(f' Creator: {meta.creator}')
@ -361,18 +384,60 @@ def call_analyse_pdf(file, verbose):
if verbose >= 2: if verbose >= 2:
print(line) print(line)
call_data_record(line, meta.modification_date,verbose) call_data_record(line, meta.modification_date,verbose, cur)
def exec_sql_file(cursor, sql_file):
statement = ''
try:
for line in open(sql_file):
if line.strip().startswith('--'): # ignore sql comment lines
continue
if line.strip().endswith(';'): # keep appending lines that don't end in ';'
statement += line
try:
cursor.execute(statement)
except mariadb.Error as e: # (OperationalError, ProgrammingError) as e:
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
statement = ''
else:
statement += line
except FileNotFoundError:
print(f'The file {path} was not found.')
except Exception as e:
print('Error: {}'.format(e), file=sys.stderr)
if __name__ == '__main__': if __name__ == '__main__':
# call_description()
args = call_parser() args = call_parser()
try: try:
filename = call_website(**vars(args)) filename = call_website(**vars(args))
if args.verbose > 1: if args.verbose > 1:
print(f'Filename: {filename}') print(f'Filename: {filename}')
call_analyse_pdf(filename,args.verbose)
sys.exit(0)
except Exception as e: except Exception as e:
print('Error: {}'.format(e), file=sys.stderr) print('Error: {}'.format(e), file=sys.stderr)
sys.exit(1) sys.exit(1)
try:
conn = mariadb.connect(
user = 'om',
password = 'oe3tkt',
host='127.0.0.1',
port=3306,
database='callbook'
)
except mariadb.Error as e:
print(f'Error connectiong to MariaDB platform: {e}')
sys.exit(5)
print(datetime.datetime.now(datetime.UTC))
# Get Cursor
cur = conn.cursor()
sql_file = '.sql_init'
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), sql_file)
exec_sql_file(cur, path)
call_analyse_pdf(filename,args.verbose,cur)
conn.commit()
cur.close()
conn.close()
sys.exit(0)