ADD mariadb database import, no addresses yet
This commit is contained in:
parent
5682220486
commit
438cccc474
324
afu/.gender
324
afu/.gender
@ -364,165 +364,165 @@ m Yannic
|
||||
m Zeljko
|
||||
m Zlatko
|
||||
m Zvonko
|
||||
w Adelheid
|
||||
w Alexandra
|
||||
w Andrea
|
||||
w Angela
|
||||
w Angelika
|
||||
w Anita
|
||||
w Anna
|
||||
w Anna-Maria
|
||||
w Anneliese
|
||||
w Annemarie
|
||||
w Astrid
|
||||
w Auguste
|
||||
w Barbara
|
||||
w Beatrice
|
||||
w Beatrix
|
||||
w Bernadette
|
||||
w Bernardine
|
||||
w Bettina
|
||||
w Bianca
|
||||
w Birgit
|
||||
w Brigitte
|
||||
w Britta
|
||||
w Carmen
|
||||
w Chiara
|
||||
w Christa
|
||||
w Christine
|
||||
w Christl
|
||||
w Cornelia
|
||||
w Durdica
|
||||
w Edeltraud
|
||||
w Elfriede
|
||||
w Elisabeth
|
||||
w Elke
|
||||
w Erdmuthe
|
||||
w Ernestine
|
||||
w Eva-Maria
|
||||
w Eveline
|
||||
w Evelyn
|
||||
w Flora
|
||||
w Franziska
|
||||
w Frederike
|
||||
w Frieda
|
||||
w Friederike
|
||||
w Gabriela
|
||||
w Gabriella
|
||||
w Gabriele
|
||||
w Gerda
|
||||
w Gerlinde
|
||||
w Gertraude
|
||||
w Gertrude
|
||||
w Gisela
|
||||
w Gudrun
|
||||
w Gunhild
|
||||
w Gustav
|
||||
w Hannelore
|
||||
w Heidelinde
|
||||
w Heidi
|
||||
w Heidrun
|
||||
w Helga
|
||||
w Hemma
|
||||
w Hermine
|
||||
w Herta
|
||||
w Hildegard
|
||||
w Ingeborg
|
||||
w Ingeburg
|
||||
w Ingrid
|
||||
w Isabel
|
||||
w Isabella
|
||||
w Isolde
|
||||
w Janet
|
||||
w Jasmin
|
||||
w Jemilla-Katalin
|
||||
w Jessica
|
||||
w Johanna
|
||||
w Josefine
|
||||
w Julia
|
||||
w Juliana
|
||||
w Jutta
|
||||
w Karin
|
||||
w Karolina
|
||||
w Karoline
|
||||
w Katharina
|
||||
w Kathrin
|
||||
w Katja
|
||||
w Katrin
|
||||
w Kerstin
|
||||
w Klaudia
|
||||
w Laila
|
||||
w Larissa
|
||||
w Leonie
|
||||
w Lieselotte
|
||||
w Ligia
|
||||
w Lisbeth
|
||||
w Lygia
|
||||
w Luisa
|
||||
w Luiza
|
||||
w Magdalena
|
||||
w Manfreda
|
||||
w Manuela
|
||||
w Margareta
|
||||
w Margarethe
|
||||
w Margot
|
||||
w Margret
|
||||
w Marianne
|
||||
w Marie-Luise
|
||||
w Marina
|
||||
w Marion
|
||||
w Marlene
|
||||
w Martha
|
||||
w Martina
|
||||
w Mathilde
|
||||
w Mechthild
|
||||
w Melanie
|
||||
w Michaela
|
||||
w Nadine
|
||||
w Natasa
|
||||
w Natascha
|
||||
w Nicole
|
||||
w Nikolitsa
|
||||
w Nina
|
||||
w Noriko
|
||||
w Olivia
|
||||
w Patrizia
|
||||
w Paulina
|
||||
w Pauline
|
||||
w Phaedra
|
||||
w Regina
|
||||
w Reinhilde
|
||||
w Renate
|
||||
w Renee
|
||||
w Rosina
|
||||
w Roswitha
|
||||
w Sabine
|
||||
w Sandra
|
||||
w Senada
|
||||
w Ricarda
|
||||
w Sieglinde
|
||||
w Silvia
|
||||
w Simone
|
||||
w Solveig
|
||||
w Sonja
|
||||
w Sophia
|
||||
w Sophie
|
||||
w Stefanie
|
||||
w Steffi
|
||||
w Stephanie
|
||||
w Susanne
|
||||
w Sybille
|
||||
w Tadeja
|
||||
w Tamara
|
||||
w Tanja
|
||||
w Tatjana
|
||||
w Theresia
|
||||
w Ulrike
|
||||
w Ursula
|
||||
w Valerie
|
||||
w Valery
|
||||
w Veronika
|
||||
w Victoria
|
||||
w Waldtraud
|
||||
w Waltraud
|
||||
w Yvonne
|
||||
f Adelheid
|
||||
f Alexandra
|
||||
f Andrea
|
||||
f Angela
|
||||
f Angelika
|
||||
f Anita
|
||||
f Anna
|
||||
f Anna-Maria
|
||||
f Anneliese
|
||||
f Annemarie
|
||||
f Astrid
|
||||
f Auguste
|
||||
f Barbara
|
||||
f Beatrice
|
||||
f Beatrix
|
||||
f Bernadette
|
||||
f Bernardine
|
||||
f Bettina
|
||||
f Bianca
|
||||
f Birgit
|
||||
f Brigitte
|
||||
f Britta
|
||||
f Carmen
|
||||
f Chiara
|
||||
f Christa
|
||||
f Christine
|
||||
f Christl
|
||||
f Cornelia
|
||||
f Durdica
|
||||
f Edeltraud
|
||||
f Elfriede
|
||||
f Elisabeth
|
||||
f Elke
|
||||
f Erdmuthe
|
||||
f Ernestine
|
||||
f Eva-Maria
|
||||
f Eveline
|
||||
f Evelyn
|
||||
f Flora
|
||||
f Franziska
|
||||
f Frederike
|
||||
f Frieda
|
||||
f Friederike
|
||||
f Gabriela
|
||||
f Gabriella
|
||||
f Gabriele
|
||||
f Gerda
|
||||
f Gerlinde
|
||||
f Gertraude
|
||||
f Gertrude
|
||||
f Gisela
|
||||
f Gudrun
|
||||
f Gunhild
|
||||
f Gustav
|
||||
f Hannelore
|
||||
f Heidelinde
|
||||
f Heidi
|
||||
f Heidrun
|
||||
f Helga
|
||||
f Hemma
|
||||
f Hermine
|
||||
f Herta
|
||||
f Hildegard
|
||||
f Ingeborg
|
||||
f Ingeburg
|
||||
f Ingrid
|
||||
f Isabel
|
||||
f Isabella
|
||||
f Isolde
|
||||
f Janet
|
||||
f Jasmin
|
||||
f Jemilla-Katalin
|
||||
f Jessica
|
||||
f Johanna
|
||||
f Josefine
|
||||
f Julia
|
||||
f Juliana
|
||||
f Jutta
|
||||
f Karin
|
||||
f Karolina
|
||||
f Karoline
|
||||
f Katharina
|
||||
f Kathrin
|
||||
f Katja
|
||||
f Katrin
|
||||
f Kerstin
|
||||
f Klaudia
|
||||
f Laila
|
||||
f Larissa
|
||||
f Leonie
|
||||
f Lieselotte
|
||||
f Ligia
|
||||
f Lisbeth
|
||||
f Lygia
|
||||
f Luisa
|
||||
f Luiza
|
||||
f Magdalena
|
||||
f Manfreda
|
||||
f Manuela
|
||||
f Margareta
|
||||
f Margarethe
|
||||
f Margot
|
||||
f Margret
|
||||
f Marianne
|
||||
f Marie-Luise
|
||||
f Marina
|
||||
f Marion
|
||||
f Marlene
|
||||
f Martha
|
||||
f Martina
|
||||
f Mathilde
|
||||
f Mechthild
|
||||
f Melanie
|
||||
f Michaela
|
||||
f Nadine
|
||||
f Natasa
|
||||
f Natascha
|
||||
f Nicole
|
||||
f Nikolitsa
|
||||
f Nina
|
||||
f Noriko
|
||||
f Olivia
|
||||
f Patrizia
|
||||
f Paulina
|
||||
f Pauline
|
||||
f Phaedra
|
||||
f Regina
|
||||
f Reinhilde
|
||||
f Renate
|
||||
f Renee
|
||||
f Rosina
|
||||
f Roswitha
|
||||
f Sabine
|
||||
f Sandra
|
||||
f Senada
|
||||
f Ricarda
|
||||
f Sieglinde
|
||||
f Silvia
|
||||
f Simone
|
||||
f Solveig
|
||||
f Sonja
|
||||
f Sophia
|
||||
f Sophie
|
||||
f Stefanie
|
||||
f Steffi
|
||||
f Stephanie
|
||||
f Susanne
|
||||
f Sybille
|
||||
f Tadeja
|
||||
f Tamara
|
||||
f Tanja
|
||||
f Tatjana
|
||||
f Theresia
|
||||
f Ulrike
|
||||
f Ursula
|
||||
f Valerie
|
||||
f Valery
|
||||
f Veronika
|
||||
f Victoria
|
||||
f Waldtraud
|
||||
f Waltraud
|
||||
f Yvonne
|
||||
37
afu/.sql_init
Normal file
37
afu/.sql_init
Normal file
@ -0,0 +1,37 @@
|
||||
-- Initialize the tables for the callbook program
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `callbook_user`(
|
||||
`id` SERIAL,
|
||||
`user_id` INT(10) UNSIGNED NOT NULL DEFAULT 0,
|
||||
`call` VARCHAR(10) NOT NULL DEFAULT '' COMMENT 'amateur radio call sign e.g. OE8ABC',
|
||||
`firstname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'forename, given name',
|
||||
`surname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'family name, last name',
|
||||
`gender` CHAR(1) NOT NULL DEFAULT '' COMMENT 'gender guessed from the first name',
|
||||
`created` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`created_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
|
||||
`modified` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`modified_by` INT(10) UNSIGNED NOT NULL DEFAULT 0,
|
||||
`active` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`inactive` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00',
|
||||
`address_id` INT(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'link to id of callbook_adress',
|
||||
PRIMARY KEY(`id`)
|
||||
) AUTO_INCREMENT=0
|
||||
DEFAULT CHARSET=utf8mb4
|
||||
COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `callbook_address`(
|
||||
`id` SERIAL,
|
||||
`location` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original location from the import',
|
||||
`address` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original address from the import',
|
||||
`postal_code` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'also known as Zip code, "Postleitzahl"',
|
||||
`org_city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
|
||||
`city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import',
|
||||
`federal_state_code` TINYINT NOT NULL DEFAULT -1
|
||||
COMMENT 'Austria: 0...outside, 1...Vienna, 2...Salzburg, 3...Lower Austria, 4...Burgenland, 5...Upper Austria, 6...Styria, 7...Tyrol, 8...Carinthia, 9...Vorarlberg',
|
||||
`lat` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'latitude',
|
||||
`long` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'longitude',
|
||||
`qth` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'Maidenhead locator, QTH locator',
|
||||
PRIMARY KEY(`id`)
|
||||
) AUTO_INCREMENT=0
|
||||
DEFAULT CHARSET=utf8mb4
|
||||
COLLATE=utf8mb4_unicode_ci;
|
||||
@ -2,13 +2,13 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import mariadb
|
||||
import sys
|
||||
import time
|
||||
import pypdf
|
||||
#from PyPDF2 import PdfReader
|
||||
from pypdf import PdfReader
|
||||
import re # regular expression
|
||||
import pandas as pd
|
||||
import datetime
|
||||
|
||||
__version__ = '1.0.0'
|
||||
__website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html'
|
||||
@ -32,7 +32,8 @@ def call_parser():
|
||||
# parser.add_argument('-s', '--server' default=__website__, required=False)
|
||||
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__))
|
||||
parser.add_argument('-v', '--verbose', action='append_const', const = 1)
|
||||
parser.add_argument('-p', '--path', default='Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
|
||||
# Rufzeichenliste_AT_Stand_010624.pdf
|
||||
parser.add_argument('-p', '--path', default='afu/Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists')
|
||||
# parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet
|
||||
parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout')
|
||||
parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format "<IP-Address>:<Port> <User> <Passwd>" or defined in .config')
|
||||
@ -176,7 +177,7 @@ def call_split_name(fullname, call, verbose):
|
||||
case 'de' | 'el':
|
||||
name = fullname[3:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
|
||||
case 'van' | 'von' :
|
||||
@ -185,12 +186,12 @@ def call_split_name(fullname, call, verbose):
|
||||
if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg"
|
||||
name = fullname[8:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
case 'della' : # Ancient Italian noble family "della Rowere"
|
||||
name = fullname[6:].split(' ',1)
|
||||
surname = surname.lower() + ' ' + name[0]
|
||||
if verbose > 0:
|
||||
if verbose > 1:
|
||||
print(f'## {fullname} --> {surname} ##')
|
||||
|
||||
if len(name) > 1:
|
||||
@ -282,7 +283,7 @@ def fix_typo(call, fullname, verbose=1):
|
||||
fix_typo.lines = None
|
||||
fix_typo.spaces = []
|
||||
|
||||
def call_data_record(line, mod_date, verbose):
|
||||
def call_data_record(line, mod_date, verbose, cur):
|
||||
|
||||
# we have to split the record with a cost-intensive regular expression
|
||||
# record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100%
|
||||
@ -311,6 +312,8 @@ def call_data_record(line, mod_date, verbose):
|
||||
address = records[3]
|
||||
permit_class = records[4]
|
||||
fullname = fix_typo(call, fullname, verbose)
|
||||
firstname = ''
|
||||
surname = ''
|
||||
# If there is a clubstation
|
||||
if is_clubstation(call):
|
||||
# Name starting with only one quotation marks e.g. " -- remove that one:
|
||||
@ -322,6 +325,7 @@ def call_data_record(line, mod_date, verbose):
|
||||
if os.path.exists(path):
|
||||
fullname = replace_substring_with_line(path, fullname, verbose)
|
||||
gender = '*'
|
||||
firstname = fullname.strip()
|
||||
elif fullname[0] == '*':
|
||||
gender = '*'
|
||||
else: # Try to split the YL or OMs Name, guess the gender
|
||||
@ -334,8 +338,27 @@ def call_data_record(line, mod_date, verbose):
|
||||
|
||||
print(f'Location: {location}, Address: {address}, Permit: {permit_class}')
|
||||
|
||||
created = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
call_data_record.cnt += 1 # increment the User_id
|
||||
user_id = call_data_record.cnt
|
||||
# print(created)
|
||||
statement = "INSERT INTO `callbook_user`(`user_id`,`call`,`firstname`,`surname`,`gender`,`created`,`modified`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
|
||||
data = (user_id,call,firstname,surname,gender,created,created)
|
||||
|
||||
def call_analyse_pdf(file, verbose):
|
||||
try:
|
||||
# cur.execute(f"INSERT INTO `callbook_user` (`call`,`firstname`,`surname`,\
|
||||
# `created`,`created_by`,`modified`,`modified_by`,`active`)\
|
||||
# VALUES ('{call}','{firstname}','{surname}','{created}','0','{created}','0','{created}');")
|
||||
# cur.execute(f'INSERT INTO `callbook_user` (`call`) VALUES ("{call}");')
|
||||
cur.execute(statement, data)
|
||||
except mariadb.Error as e:
|
||||
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
|
||||
call_data_record.cnt = 0
|
||||
|
||||
def call_analyse_pdf(file, verbose, cur):
|
||||
|
||||
# Define a regular expression to match tables
|
||||
|
||||
@ -343,7 +366,7 @@ def call_analyse_pdf(file, verbose):
|
||||
meta = reader.metadata
|
||||
if verbose:
|
||||
print(verbose)
|
||||
print(' Pages:', len(reader.pages))
|
||||
print(' Pages:', len(reader.pages))
|
||||
# All of the following could be None!
|
||||
print(f' Author: {meta.author}')
|
||||
print(f' Creator: {meta.creator}')
|
||||
@ -361,18 +384,60 @@ def call_analyse_pdf(file, verbose):
|
||||
|
||||
if verbose >= 2:
|
||||
print(line)
|
||||
call_data_record(line, meta.modification_date,verbose)
|
||||
call_data_record(line, meta.modification_date,verbose, cur)
|
||||
|
||||
def exec_sql_file(cursor, sql_file):
|
||||
statement = ''
|
||||
try:
|
||||
for line in open(sql_file):
|
||||
if line.strip().startswith('--'): # ignore sql comment lines
|
||||
continue
|
||||
if line.strip().endswith(';'): # keep appending lines that don't end in ';'
|
||||
statement += line
|
||||
try:
|
||||
cursor.execute(statement)
|
||||
except mariadb.Error as e: # (OperationalError, ProgrammingError) as e:
|
||||
print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}')
|
||||
statement = ''
|
||||
else:
|
||||
statement += line
|
||||
except FileNotFoundError:
|
||||
print(f'The file {path} was not found.')
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# call_description()
|
||||
args = call_parser()
|
||||
|
||||
try:
|
||||
filename = call_website(**vars(args))
|
||||
if args.verbose > 1:
|
||||
print(f'Filename: {filename}')
|
||||
call_analyse_pdf(filename,args.verbose)
|
||||
sys.exit(0)
|
||||
|
||||
except Exception as e:
|
||||
print('Error: {}'.format(e), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
conn = mariadb.connect(
|
||||
user = 'om',
|
||||
password = 'oe3tkt',
|
||||
host='127.0.0.1',
|
||||
port=3306,
|
||||
database='callbook'
|
||||
)
|
||||
except mariadb.Error as e:
|
||||
print(f'Error connectiong to MariaDB platform: {e}')
|
||||
sys.exit(5)
|
||||
|
||||
print(datetime.datetime.now(datetime.UTC))
|
||||
# Get Cursor
|
||||
cur = conn.cursor()
|
||||
sql_file = '.sql_init'
|
||||
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), sql_file)
|
||||
exec_sql_file(cur, path)
|
||||
|
||||
call_analyse_pdf(filename,args.verbose,cur)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
sys.exit(0)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user