ADD mariadb database import, no addresses yet

This commit is contained in:
Thomas Kuschel 2024-06-19 19:51:08 +02:00
parent 5682220486
commit 438cccc474
3 changed files with 278 additions and 176 deletions

View File

@ -364,165 +364,165 @@ m Yannic
m Zeljko
m Zlatko
m Zvonko
w Adelheid
w Alexandra
w Andrea
w Angela
w Angelika
w Anita
w Anna
w Anna-Maria
w Anneliese
w Annemarie
w Astrid
w Auguste
w Barbara
w Beatrice
w Beatrix
w Bernadette
w Bernardine
w Bettina
w Bianca
w Birgit
w Brigitte
w Britta
w Carmen
w Chiara
w Christa
w Christine
w Christl
w Cornelia
w Durdica
w Edeltraud
w Elfriede
w Elisabeth
w Elke
w Erdmuthe
w Ernestine
w Eva-Maria
w Eveline
w Evelyn
w Flora
w Franziska
w Frederike
w Frieda
w Friederike
w Gabriela
w Gabriella
w Gabriele
w Gerda
w Gerlinde
w Gertraude
w Gertrude
w Gisela
w Gudrun
w Gunhild
w Gustav
w Hannelore
w Heidelinde
w Heidi
w Heidrun
w Helga
w Hemma
w Hermine
w Herta
w Hildegard
w Ingeborg
w Ingeburg
w Ingrid
w Isabel
w Isabella
w Isolde
w Janet
w Jasmin
w Jemilla-Katalin
w Jessica
w Johanna
w Josefine
w Julia
w Juliana
w Jutta
w Karin
w Karolina
w Karoline
w Katharina
w Kathrin
w Katja
w Katrin
w Kerstin
w Klaudia
w Laila
w Larissa
w Leonie
w Lieselotte
w Ligia
w Lisbeth
w Lygia
w Luisa
w Luiza
w Magdalena
w Manfreda
w Manuela
w Margareta
w Margarethe
w Margot
w Margret
w Marianne
w Marie-Luise
w Marina
w Marion
w Marlene
w Martha
w Martina
w Mathilde
w Mechthild
w Melanie
w Michaela
w Nadine
w Natasa
w Natascha
w Nicole
w Nikolitsa
w Nina
w Noriko
w Olivia
w Patrizia
w Paulina
w Pauline
w Phaedra
w Regina
w Reinhilde
w Renate
w Renee
w Rosina
w Roswitha
w Sabine
w Sandra
w Senada
w Ricarda
w Sieglinde
w Silvia
w Simone
w Solveig
w Sonja
w Sophia
w Sophie
w Stefanie
w Steffi
w Stephanie
w Susanne
w Sybille
w Tadeja
w Tamara
w Tanja
w Tatjana
w Theresia
w Ulrike
w Ursula
w Valerie
w Valery
w Veronika
w Victoria
w Waldtraud
w Waltraud
w Yvonne
f Adelheid
f Alexandra
f Andrea
f Angela
f Angelika
f Anita
f Anna
f Anna-Maria
f Anneliese
f Annemarie
f Astrid
f Auguste
f Barbara
f Beatrice
f Beatrix
f Bernadette
f Bernardine
f Bettina
f Bianca
f Birgit
f Brigitte
f Britta
f Carmen
f Chiara
f Christa
f Christine
f Christl
f Cornelia
f Durdica
f Edeltraud
f Elfriede
f Elisabeth
f Elke
f Erdmuthe
f Ernestine
f Eva-Maria
f Eveline
f Evelyn
f Flora
f Franziska
f Frederike
f Frieda
f Friederike
f Gabriela
f Gabriella
f Gabriele
f Gerda
f Gerlinde
f Gertraude
f Gertrude
f Gisela
f Gudrun
f Gunhild
f Gustav
f Hannelore
f Heidelinde
f Heidi
f Heidrun
f Helga
f Hemma
f Hermine
f Herta
f Hildegard
f Ingeborg
f Ingeburg
f Ingrid
f Isabel
f Isabella
f Isolde
f Janet
f Jasmin
f Jemilla-Katalin
f Jessica
f Johanna
f Josefine
f Julia
f Juliana
f Jutta
f Karin
f Karolina
f Karoline
f Katharina
f Kathrin
f Katja
f Katrin
f Kerstin
f Klaudia
f Laila
f Larissa
f Leonie
f Lieselotte
f Ligia
f Lisbeth
f Lygia
f Luisa
f Luiza
f Magdalena
f Manfreda
f Manuela
f Margareta
f Margarethe
f Margot
f Margret
f Marianne
f Marie-Luise
f Marina
f Marion
f Marlene
f Martha
f Martina
f Mathilde
f Mechthild
f Melanie
f Michaela
f Nadine
f Natasa
f Natascha
f Nicole
f Nikolitsa
f Nina
f Noriko
f Olivia
f Patrizia
f Paulina
f Pauline
f Phaedra
f Regina
f Reinhilde
f Renate
f Renee
f Rosina
f Roswitha
f Sabine
f Sandra
f Senada
f Ricarda
f Sieglinde
f Silvia
f Simone
f Solveig
f Sonja
f Sophia
f Sophie
f Stefanie
f Steffi
f Stephanie
f Susanne
f Sybille
f Tadeja
f Tamara
f Tanja
f Tatjana
f Theresia
f Ulrike
f Ursula
f Valerie
f Valery
f Veronika
f Victoria
f Waldtraud
f Waltraud
f Yvonne

37
afu/.sql_init Normal file
View File

@ -0,0 +1,37 @@
-- Initialize the tables for the callbook program
CREATE TABLE IF NOT EXISTS `callbook_user`(
`id` SERIAL,
`user_id` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`call` VARCHAR(10) NOT NULL DEFAULT '' COMMENT 'amateur radio call sign e.g. OE8ABC',
`firstname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'forename, given name',
`surname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'family name, last name',
`gender` CHAR(1) NOT NULL DEFAULT '' COMMENT 'gender guessed from the first name',
`created` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`created_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`modified` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`modified_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
`active` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`inactive` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
`address_id` INT(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'link to id of callbook_adress',
PRIMARY KEY(`id`)
) AUTO_INCREMENT=0
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `callbook_address`(
`id` SERIAL,
`location` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original location from the import',
`address` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original address from the import',
`postal_code` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'also known as Zip code, "Postleitzahl"',
`org_city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
`city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
`federal_state_code` TINYINT NOT NULL DEFAULT -1
COMMENT 'Austria: 0...outside, 1...Vienna, 2...Salzburg, 3...Lower Austria, 4...Burgenland, 5...Upper Austria, 6...Styria, 7...Tyrol, 8...Carinthia, 9...Vorarlberg',
`lat` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'latitude',
`long` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'longitude',
`qth` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'Maidenhead locator, QTH locator',
PRIMARY KEY(`id`)
) AUTO_INCREMENT=0
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_unicode_ci;

View File

@ -2,13 +2,13 @@
import argparse
import os
import mariadb
import sys
import time
import pypdf
#from PyPDF2 import PdfReader
from pypdf import PdfReader
import re # regular expression
import pandas as pd
import datetime
__version__ = '1.0.0'
__website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html'
@ -32,7 +32,8 @@ def call_parser():
# parser.add_argument('-s', '--server' default=__website__, required=False)
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__))
parser.add_argument('-v', '--verbose', action='append_const', const = 1)
parser.add_argument('-p', '--path', default='Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
# Rufzeichenliste_AT_Stand_010624.pdf
parser.add_argument('-p', '--path', default='afu/Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
# parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet
parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout')
parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format "<IP-Address>:<Port> <User> <Passwd>" or defined in .config')
@ -176,7 +177,7 @@ def call_split_name(fullname, call, verbose):
case 'de' | 'el':
name = fullname[3:].split(' ',1)
surname = surname.lower() + ' ' + name[0]
if verbose > 0:
if verbose > 1:
print(f'## {fullname} --> {surname} ##')
case 'van' | 'von' :
@ -185,12 +186,12 @@ def call_split_name(fullname, call, verbose):
if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg"
name = fullname[8:].split(' ',1)
surname = surname.lower() + ' ' + name[0]
if verbose > 0:
if verbose > 1:
print(f'## {fullname} --> {surname} ##')
case 'della' : # Ancient Italian noble family "della Rowere"
name = fullname[6:].split(' ',1)
surname = surname.lower() + ' ' + name[0]
if verbose > 0:
if verbose > 1:
print(f'## {fullname} --> {surname} ##')
if len(name) > 1:
@ -282,7 +283,7 @@ def fix_typo(call, fullname, verbose=1):
fix_typo.lines = None
fix_typo.spaces = []
def call_data_record(line, mod_date, verbose):
def call_data_record(line, mod_date, verbose, cur):
# we have to split the record with a cost-intensive regular expression
# record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100%
@ -311,6 +312,8 @@ def call_data_record(line, mod_date, verbose):
address = records[3]
permit_class = records[4]
fullname = fix_typo(call, fullname, verbose)
firstname = ''
surname = ''
# If there is a clubstation
if is_clubstation(call):
# Name starting with only one quotation marks e.g. " -- remove that one:
@ -322,6 +325,7 @@ def call_data_record(line, mod_date, verbose):
if os.path.exists(path):
fullname = replace_substring_with_line(path, fullname, verbose)
gender = '*'
firstname = fullname.strip()
elif fullname[0] == '*':
gender = '*'
else: # Try to split the YL or OMs Name, guess the gender
@ -334,8 +338,27 @@ def call_data_record(line, mod_date, verbose):
print(f'Location: {location}, Address: {address}, Permit: {permit_class}')
created = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
call_data_record.cnt += 1 # increment the User_id
user_id = call_data_record.cnt
# print(created)
statement = "INSERT INTO `callbook_user`(`user_id`,`call`,`firstname`,`surname`,`gender`,`created`,`modified`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
data = (user_id,call,firstname,surname,gender,created,created)
def call_analyse_pdf(file, verbose):
try:
# cur.execute(f"INSERT INTO `callbook_user` (`call`,`firstname`,`surname`,\
# `created`,`created_by`,`modified`,`modified_by`,`active`)\
# VALUES ('{call}','{firstname}','{surname}','{created}','0','{created}','0','{created}');")
# cur.execute(f'INSERT INTO `callbook_user` (`call`) VALUES ("{call}");')
cur.execute(statement, data)
except mariadb.Error as e:
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
except Exception as e:
print('Error: {}'.format(e), file=sys.stderr)
call_data_record.cnt = 0
def call_analyse_pdf(file, verbose, cur):
# Define a regular expression to match tables
@ -361,18 +384,60 @@ def call_analyse_pdf(file, verbose):
if verbose >= 2:
print(line)
call_data_record(line, meta.modification_date,verbose)
call_data_record(line, meta.modification_date,verbose, cur)
def exec_sql_file(cursor, sql_file):
statement = ''
try:
for line in open(sql_file):
if line.strip().startswith('--'): # ignore sql comment lines
continue
if line.strip().endswith(';'): # keep appending lines that don't end in ';'
statement += line
try:
cursor.execute(statement)
except mariadb.Error as e: # (OperationalError, ProgrammingError) as e:
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
statement = ''
else:
statement += line
except FileNotFoundError:
print(f'The file {path} was not found.')
except Exception as e:
print('Error: {}'.format(e), file=sys.stderr)
if __name__ == '__main__':
# call_description()
args = call_parser()
try:
filename = call_website(**vars(args))
if args.verbose > 1:
print(f'Filename: {filename}')
call_analyse_pdf(filename,args.verbose)
sys.exit(0)
except Exception as e:
print('Error: {}'.format(e), file=sys.stderr)
sys.exit(1)
try:
conn = mariadb.connect(
user = 'om',
password = 'oe3tkt',
host='127.0.0.1',
port=3306,
database='callbook'
)
except mariadb.Error as e:
print(f'Error connectiong to MariaDB platform: {e}')
sys.exit(5)
print(datetime.datetime.now(datetime.UTC))
# Get Cursor
cur = conn.cursor()
sql_file = '.sql_init'
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), sql_file)
exec_sql_file(cur, path)
call_analyse_pdf(filename,args.verbose,cur)
conn.commit()
cur.close()
conn.close()
sys.exit(0)