diff --git a/afu/.gender b/afu/.gender index 9c5a2f6..d9ad693 100644 --- a/afu/.gender +++ b/afu/.gender @@ -364,165 +364,165 @@ m Yannic m Zeljko m Zlatko m Zvonko -w Adelheid -w Alexandra -w Andrea -w Angela -w Angelika -w Anita -w Anna -w Anna-Maria -w Anneliese -w Annemarie -w Astrid -w Auguste -w Barbara -w Beatrice -w Beatrix -w Bernadette -w Bernardine -w Bettina -w Bianca -w Birgit -w Brigitte -w Britta -w Carmen -w Chiara -w Christa -w Christine -w Christl -w Cornelia -w Durdica -w Edeltraud -w Elfriede -w Elisabeth -w Elke -w Erdmuthe -w Ernestine -w Eva-Maria -w Eveline -w Evelyn -w Flora -w Franziska -w Frederike -w Frieda -w Friederike -w Gabriela -w Gabriella -w Gabriele -w Gerda -w Gerlinde -w Gertraude -w Gertrude -w Gisela -w Gudrun -w Gunhild -w Gustav -w Hannelore -w Heidelinde -w Heidi -w Heidrun -w Helga -w Hemma -w Hermine -w Herta -w Hildegard -w Ingeborg -w Ingeburg -w Ingrid -w Isabel -w Isabella -w Isolde -w Janet -w Jasmin -w Jemilla-Katalin -w Jessica -w Johanna -w Josefine -w Julia -w Juliana -w Jutta -w Karin -w Karolina -w Karoline -w Katharina -w Kathrin -w Katja -w Katrin -w Kerstin -w Klaudia -w Laila -w Larissa -w Leonie -w Lieselotte -w Ligia -w Lisbeth -w Lygia -w Luisa -w Luiza -w Magdalena -w Manfreda -w Manuela -w Margareta -w Margarethe -w Margot -w Margret -w Marianne -w Marie-Luise -w Marina -w Marion -w Marlene -w Martha -w Martina -w Mathilde -w Mechthild -w Melanie -w Michaela -w Nadine -w Natasa -w Natascha -w Nicole -w Nikolitsa -w Nina -w Noriko -w Olivia -w Patrizia -w Paulina -w Pauline -w Phaedra -w Regina -w Reinhilde -w Renate -w Renee -w Rosina -w Roswitha -w Sabine -w Sandra -w Senada -w Ricarda -w Sieglinde -w Silvia -w Simone -w Solveig -w Sonja -w Sophia -w Sophie -w Stefanie -w Steffi -w Stephanie -w Susanne -w Sybille -w Tadeja -w Tamara -w Tanja -w Tatjana -w Theresia -w Ulrike -w Ursula -w Valerie -w Valery -w Veronika -w Victoria -w Waldtraud -w Waltraud -w Yvonne \ No newline at end of file +f Adelheid +f Alexandra +f Andrea +f Angela +f Angelika +f Anita +f Anna +f Anna-Maria +f Anneliese +f Annemarie +f Astrid +f Auguste +f Barbara +f Beatrice +f Beatrix +f Bernadette +f Bernardine +f Bettina +f Bianca +f Birgit +f Brigitte +f Britta +f Carmen +f Chiara +f Christa +f Christine +f Christl +f Cornelia +f Durdica +f Edeltraud +f Elfriede +f Elisabeth +f Elke +f Erdmuthe +f Ernestine +f Eva-Maria +f Eveline +f Evelyn +f Flora +f Franziska +f Frederike +f Frieda +f Friederike +f Gabriela +f Gabriella +f Gabriele +f Gerda +f Gerlinde +f Gertraude +f Gertrude +f Gisela +f Gudrun +f Gunhild +f Gustav +f Hannelore +f Heidelinde +f Heidi +f Heidrun +f Helga +f Hemma +f Hermine +f Herta +f Hildegard +f Ingeborg +f Ingeburg +f Ingrid +f Isabel +f Isabella +f Isolde +f Janet +f Jasmin +f Jemilla-Katalin +f Jessica +f Johanna +f Josefine +f Julia +f Juliana +f Jutta +f Karin +f Karolina +f Karoline +f Katharina +f Kathrin +f Katja +f Katrin +f Kerstin +f Klaudia +f Laila +f Larissa +f Leonie +f Lieselotte +f Ligia +f Lisbeth +f Lygia +f Luisa +f Luiza +f Magdalena +f Manfreda +f Manuela +f Margareta +f Margarethe +f Margot +f Margret +f Marianne +f Marie-Luise +f Marina +f Marion +f Marlene +f Martha +f Martina +f Mathilde +f Mechthild +f Melanie +f Michaela +f Nadine +f Natasa +f Natascha +f Nicole +f Nikolitsa +f Nina +f Noriko +f Olivia +f Patrizia +f Paulina +f Pauline +f Phaedra +f Regina +f Reinhilde +f Renate +f Renee +f Rosina +f Roswitha +f Sabine +f Sandra +f Senada +f Ricarda +f Sieglinde +f Silvia +f Simone +f Solveig +f Sonja +f Sophia +f Sophie +f Stefanie +f Steffi +f Stephanie +f Susanne +f Sybille +f Tadeja +f Tamara +f Tanja +f Tatjana +f Theresia +f Ulrike +f Ursula +f Valerie +f Valery +f Veronika +f Victoria +f Waldtraud +f Waltraud +f Yvonne \ No newline at end of file diff --git a/afu/.sql_init b/afu/.sql_init new file mode 100644 index 0000000..589ba1a --- /dev/null +++ b/afu/.sql_init @@ -0,0 +1,37 @@ +-- Initialize the tables for the callbook program + +CREATE TABLE IF NOT EXISTS `callbook_user`( + `id` SERIAL, + `user_id` INT(10) UNSIGNED NOT NULL DEFAULT 0, + `call` VARCHAR(10) NOT NULL DEFAULT '' COMMENT 'amateur radio call sign e.g. OE8ABC', + `firstname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'forename, given name', + `surname` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'family name, last name', + `gender` CHAR(1) NOT NULL DEFAULT '' COMMENT 'gender guessed from the first name', + `created` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', + `created_by` INT(10) UNSIGNED NOT NULL DEFAULT 0, + `modified` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', + `modified_by` INT(10) UNSIGNED NOT NULL DEFAULT 0, + `active` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', + `inactive` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', + `address_id` INT(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'link to id of callbook_adress', + PRIMARY KEY(`id`) + ) AUTO_INCREMENT=0 + DEFAULT CHARSET=utf8mb4 + COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS `callbook_address`( + `id` SERIAL, + `location` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original location from the import', + `address` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original address from the import', + `postal_code` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'also known as Zip code, "Postleitzahl"', + `org_city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import', + `city` VARCHAR(200) NOT NULL DEFAULT '' COMMENT 'original city from the import', + `federal_state_code` TINYINT NOT NULL DEFAULT -1 + COMMENT 'Austria: 0...outside, 1...Vienna, 2...Salzburg, 3...Lower Austria, 4...Burgenland, 5...Upper Austria, 6...Styria, 7...Tyrol, 8...Carinthia, 9...Vorarlberg', + `lat` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'latitude', + `long` DOUBLE NOT NULL DEFAULT 0.0 COMMENT 'longitude', + `qth` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'Maidenhead locator, QTH locator', + PRIMARY KEY(`id`) + ) AUTO_INCREMENT=0 + DEFAULT CHARSET=utf8mb4 + COLLATE=utf8mb4_unicode_ci; diff --git a/afu/callbook.py b/afu/callbook.py index 334f97f..6264ea1 100755 --- a/afu/callbook.py +++ b/afu/callbook.py @@ -2,13 +2,13 @@ import argparse import os +import mariadb import sys import time import pypdf -#from PyPDF2 import PdfReader from pypdf import PdfReader import re # regular expression -import pandas as pd +import datetime __version__ = '1.0.0' __website__ = 'https://www.fb.gv.at/Funk/amateurfunkdienst.html' @@ -32,7 +32,8 @@ def call_parser(): # parser.add_argument('-s', '--server' default=__website__, required=False) parser.add_argument('-V', '--version', action='version', version='{} {}'.format(os.path.split(__file__)[1],__version__)) parser.add_argument('-v', '--verbose', action='append_const', const = 1) - parser.add_argument('-p', '--path', default='Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists') + # Rufzeichenliste_AT_Stand_010624.pdf + parser.add_argument('-p', '--path', default='afu/Rufzeichenliste_AT_Stand_010624.pdf', help= 'skip the download if the specified path to a PDF file exists') # parser.add_argument('-t', '--type', default='' , help='specify the output, supported types are [ CSV | JSON ]') # not implemented yet parser.add_argument('-o', '--output', default='', help='specify the file where the data are written to, default stdout') parser.add_argument('-m', '--mariadb', help='SQL interface to MariaDB (MySql) format ": " or defined in .config') @@ -176,7 +177,7 @@ def call_split_name(fullname, call, verbose): case 'de' | 'el': name = fullname[3:].split(' ',1) surname = surname.lower() + ' ' + name[0] - if verbose > 0: + if verbose > 1: print(f'## {fullname} --> {surname} ##') case 'van' | 'von' : @@ -185,12 +186,12 @@ def call_split_name(fullname, call, verbose): if surname.lower() in ['van der', 'von der', 'van den']: # e.g. "van der Meulen", "Walther von der Vogelweide", "Annie van den Berg" name = fullname[8:].split(' ',1) surname = surname.lower() + ' ' + name[0] - if verbose > 0: + if verbose > 1: print(f'## {fullname} --> {surname} ##') case 'della' : # Ancient Italian noble family "della Rowere" name = fullname[6:].split(' ',1) surname = surname.lower() + ' ' + name[0] - if verbose > 0: + if verbose > 1: print(f'## {fullname} --> {surname} ##') if len(name) > 1: @@ -282,7 +283,7 @@ def fix_typo(call, fullname, verbose=1): fix_typo.lines = None fix_typo.spaces = [] -def call_data_record(line, mod_date, verbose): +def call_data_record(line, mod_date, verbose, cur): # we have to split the record with a cost-intensive regular expression # record = re.split('OE[0-9][A-Z]{1,3}[ \t]{3,20}',line) # this does not work 100% @@ -311,6 +312,8 @@ def call_data_record(line, mod_date, verbose): address = records[3] permit_class = records[4] fullname = fix_typo(call, fullname, verbose) + firstname = '' + surname = '' # If there is a clubstation if is_clubstation(call): # Name starting with only one quotation marks e.g. " -- remove that one: @@ -322,6 +325,7 @@ def call_data_record(line, mod_date, verbose): if os.path.exists(path): fullname = replace_substring_with_line(path, fullname, verbose) gender = '*' + firstname = fullname.strip() elif fullname[0] == '*': gender = '*' else: # Try to split the YL or OMs Name, guess the gender @@ -334,8 +338,27 @@ def call_data_record(line, mod_date, verbose): print(f'Location: {location}, Address: {address}, Permit: {permit_class}') + created = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + call_data_record.cnt += 1 # increment the User_id + user_id = call_data_record.cnt + # print(created) + statement = "INSERT INTO `callbook_user`(`user_id`,`call`,`firstname`,`surname`,`gender`,`created`,`modified`) VALUES (%s, %s, %s, %s, %s, %s, %s)" + data = (user_id,call,firstname,surname,gender,created,created) -def call_analyse_pdf(file, verbose): + try: + # cur.execute(f"INSERT INTO `callbook_user` (`call`,`firstname`,`surname`,\ + # `created`,`created_by`,`modified`,`modified_by`,`active`)\ + # VALUES ('{call}','{firstname}','{surname}','{created}','0','{created}','0','{created}');") + # cur.execute(f'INSERT INTO `callbook_user` (`call`) VALUES ("{call}");') + cur.execute(statement, data) + except mariadb.Error as e: + print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}') + except Exception as e: + print('Error: {}'.format(e), file=sys.stderr) + +call_data_record.cnt = 0 + +def call_analyse_pdf(file, verbose, cur): # Define a regular expression to match tables @@ -343,7 +366,7 @@ def call_analyse_pdf(file, verbose): meta = reader.metadata if verbose: print(verbose) - print(' Pages:', len(reader.pages)) + print(' Pages:', len(reader.pages)) # All of the following could be None! print(f' Author: {meta.author}') print(f' Creator: {meta.creator}') @@ -361,18 +384,60 @@ def call_analyse_pdf(file, verbose): if verbose >= 2: print(line) - call_data_record(line, meta.modification_date,verbose) + call_data_record(line, meta.modification_date,verbose, cur) + +def exec_sql_file(cursor, sql_file): + statement = '' + try: + for line in open(sql_file): + if line.strip().startswith('--'): # ignore sql comment lines + continue + if line.strip().endswith(';'): # keep appending lines that don't end in ';' + statement += line + try: + cursor.execute(statement) + except mariadb.Error as e: # (OperationalError, ProgrammingError) as e: + print(f'\n[WARN] MySQLError during execute statement\n\tArgs: {e.args}') + statement = '' + else: + statement += line + except FileNotFoundError: + print(f'The file {path} was not found.') + except Exception as e: + print('Error: {}'.format(e), file=sys.stderr) if __name__ == '__main__': - # call_description() args = call_parser() - try: filename = call_website(**vars(args)) if args.verbose > 1: print(f'Filename: {filename}') - call_analyse_pdf(filename,args.verbose) - sys.exit(0) + except Exception as e: print('Error: {}'.format(e), file=sys.stderr) sys.exit(1) + + try: + conn = mariadb.connect( + user = 'om', + password = 'oe3tkt', + host='127.0.0.1', + port=3306, + database='callbook' + ) + except mariadb.Error as e: + print(f'Error connectiong to MariaDB platform: {e}') + sys.exit(5) + + print(datetime.datetime.now(datetime.UTC)) + # Get Cursor + cur = conn.cursor() + sql_file = '.sql_init' + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), sql_file) + exec_sql_file(cur, path) + + call_analyse_pdf(filename,args.verbose,cur) + conn.commit() + cur.close() + conn.close() + sys.exit(0)