133 lines
3.6 KiB
Python
133 lines
3.6 KiB
Python
import sqlalchemy as db #somehow pyodbc does not work in container
|
|
import numpy as np
|
|
from sqlalchemy.sql import text
|
|
from sqlalchemy_utils import database_exists, create_database
|
|
import argparse
|
|
import time
|
|
|
|
|
|
import source.DataBank.createDatabank as createDB
|
|
import source.DataBank.fillDatabank as fillDB
|
|
|
|
|
|
'''
|
|
read data of Dobby from volume
|
|
'''
|
|
def loadDobbyData():
|
|
personsArray = np.loadtxt("/var/lib/data/fullDobby.csv", delimiter=",", dtype='U60') #important that we load strings
|
|
print(personsArray)
|
|
|
|
Mitarbeiter = np.zeros((len(personsArray), 7), dtype='U60') #need to establish this array as array of strings,
|
|
#to be able to save strings and not floats there
|
|
for i in range(len(personsArray)):
|
|
Mitarbeiter[i,0] = personsArray[i,0] #Kuerzel
|
|
|
|
nameParts = personsArray[i,1].split() #split the full name
|
|
nachName = nameParts[len(nameParts) - 1] #last word is last Name
|
|
vorName = " ".join(nameParts[:-1]) #all words before is first name
|
|
|
|
Mitarbeiter[i,1] = vorName #Vorname
|
|
Mitarbeiter[i,2] = nachName
|
|
|
|
Mitarbeiter[i,3] = 'hh'
|
|
Mitarbeiter[i,4] = '2022-01-01'
|
|
Mitarbeiter[i,5] = 'mail@example.com'
|
|
Mitarbeiter[i,6] = personsArray[i,2]
|
|
print(Mitarbeiter)
|
|
|
|
return Mitarbeiter
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(db_user, db_pass, db_host, db_port, db_name):
|
|
'''
|
|
create argparse instance: --init if we want to create tables
|
|
'''
|
|
parser = argparse.ArgumentParser()
|
|
#stores true if we start like: python main.py --init
|
|
parser.add_argument('-i', '--init', action='store_true', default=False, dest='boolean_init')
|
|
#name of argument --init is boolean_init
|
|
init = parser.parse_args().boolean_init
|
|
print(init)
|
|
|
|
|
|
'''
|
|
connect to dataBank
|
|
'''
|
|
#build our engine to connect and execute in our databank
|
|
db_string = 'postgresql://{}:{}@{}:{}/{}'.format(db_user, db_pass, db_host, db_port, db_name)
|
|
engine = db.create_engine(db_string, echo_pool=True)
|
|
#build the database itself, if it does not exist already
|
|
if not database_exists(engine.url):
|
|
print('db did not exist before')
|
|
create_database(engine.url)
|
|
|
|
'''
|
|
wait some time, so all data from the other containers can be scrapped
|
|
and saved
|
|
therefore the main.py can load all data and save it in DB
|
|
'''
|
|
#TODO: need dependency and NOT fixed sleep time
|
|
time.sleep(600)
|
|
|
|
|
|
'''
|
|
load Data scraped from Dobby
|
|
'''
|
|
Mitarbeiter = loadDobbyData()
|
|
mitarbeiterAnzahl = len(Mitarbeiter)
|
|
|
|
|
|
|
|
|
|
'''
|
|
Now all SQL commands
|
|
'''
|
|
with engine.connect() as connection:
|
|
if init:
|
|
createDB.createMitarbeiter(connection)
|
|
print('create')
|
|
|
|
|
|
try:
|
|
for i in range(mitarbeiterAnzahl):
|
|
fillDB.inputMitarbeiter(connection, Mitarbeiter[i])
|
|
except:
|
|
print("{} was filled in allready".format(Mitarbeiter[i][0]))
|
|
#createDB.dropMitarbeiter(connection)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
'''
|
|
set values to connect to databank
|
|
'''
|
|
#NOTE: you maybe have to change db_host and db_name depending on your machine
|
|
#TODO: variables in docker compose
|
|
db_user = 'root'
|
|
db_pass = 'root'
|
|
db_host = 'pg_container_test_vimes' #at least should get postgres container ID automatically
|
|
#but not always working
|
|
db_port = '5432'
|
|
db_name = 'test_db_vimes'
|
|
|
|
main(db_user, db_pass, db_host, db_port, db_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|