import sqlalchemy as db #somehow pyodbc does not work in container import numpy as np from sqlalchemy.sql import text from sqlalchemy_utils import database_exists, create_database import argparse import time import source.DataBank.createDatabank as createDB import source.DataBank.fillDatabank as fillDB ''' read data of Dobby from volume ''' def loadDobbyData(): personsArray = np.loadtxt("/var/lib/data/fullDobby.csv", delimiter=",", dtype='U60') #important that we load strings print(personsArray) Mitarbeiter = np.zeros((len(personsArray), 7), dtype='U60') #need to establish this array as array of strings, #to be able to save strings and not floats there for i in range(len(personsArray)): Mitarbeiter[i,0] = personsArray[i,0] #Kuerzel nameParts = personsArray[i,1].split() #split the full name nachName = nameParts[len(nameParts) - 1] #last word is last Name vorName = " ".join(nameParts[:-1]) #all words before is first name Mitarbeiter[i,1] = vorName #Vorname Mitarbeiter[i,2] = nachName Mitarbeiter[i,3] = 'hh' Mitarbeiter[i,4] = '2022-01-01' Mitarbeiter[i,5] = 'mail@example.com' Mitarbeiter[i,6] = personsArray[i,2] print(Mitarbeiter) return Mitarbeiter def main(db_user, db_pass, db_host, db_port, db_name): ''' create argparse instance: --init if we want to create tables ''' parser = argparse.ArgumentParser() #stores true if we start like: python main.py --init parser.add_argument('-i', '--init', action='store_true', default=False, dest='boolean_init') #name of argument --init is boolean_init init = parser.parse_args().boolean_init print(init) ''' connect to dataBank ''' #build our engine to connect and execute in our databank db_string = 'postgresql://{}:{}@{}:{}/{}'.format(db_user, db_pass, db_host, db_port, db_name) engine = db.create_engine(db_string, echo_pool=True) #build the database itself, if it does not exist already if not database_exists(engine.url): print('db did not exist before') create_database(engine.url) ''' wait some time, so all data from the other containers can be scrapped and saved therefore the main.py can load all data and save it in DB ''' #TODO: need dependency and NOT fixed sleep time time.sleep(600) ''' load Data scraped from Dobby ''' Mitarbeiter = loadDobbyData() mitarbeiterAnzahl = len(Mitarbeiter) ''' Now all SQL commands ''' with engine.connect() as connection: if init: createDB.createMitarbeiter(connection) print('create') try: for i in range(mitarbeiterAnzahl): fillDB.inputMitarbeiter(connection, Mitarbeiter[i]) except: print("{} was filled in allready".format(Mitarbeiter[i][0])) #createDB.dropMitarbeiter(connection) if __name__ == '__main__': ''' set values to connect to databank ''' #NOTE: you maybe have to change db_host and db_name depending on your machine #TODO: variables in docker compose db_user = 'root' db_pass = 'root' db_host = 'pg_container_test_vimes' #at least should get postgres container ID automatically #but not always working db_port = '5432' db_name = 'test_db_vimes' main(db_user, db_pass, db_host, db_port, db_name)