from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from neo4j import GraphDatabase #some notes: #the links to find elements can be found with: #right click on browser and choose Untersuchen(Q) #shows all elements => find the needed table by hand #define the driver for the databank uri = "bolt://127.0.0.1:7687" userName = "neo4j" password = "org" graphDB_Driver = GraphDatabase.driver(uri, auth=(userName, password)) #find names of each Group of PPI.X on Dobby def findGroupNames(): groupsPpiX = [] #define a driver = mousepointer using Chrome as Browser (Firefox did not work) driver = webdriver.Chrome() #define the link to access and access it projectPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte" driver.get(projectPath) #find the table with all projects (the lower table) table = driver.find_element("xpath", "/html/body/div/div/table[2]") #go through each row in that table for row in table.find_elements("xpath", ".//tr[not(position()=1)]"): try: groupId = row.find_element("xpath", ".//td[1]").text #first column in table if Group groupDesc = row.find_element("xpath", ".//td[2]").text #second is Description if groupId[:8] == 'prj_ppix': #only choose groups starting like that = from PPI.X groupsPpiX.append([groupId, groupDesc]) except NoSuchElementException: print("NoSuchElementException") driver.close() print(groupsPpiX) return groupsPpiX #find all persons working at PPI.X #save names and short notation in def getPersons(groupsNames): persons = set() driver = webdriver.Chrome() groupPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?group=" for group, groupDesc in groupsNames: driver.get(groupPath + group) table = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody") personsInGroup = [] #collect all persons (name+short notation) in that group we are locking at now #go though each row in table with short notation and name #do not consider first row (tr[not(position()=1)]) since caption only for row in table.find_elements("xpath", ".//tr[not(position()=1)]"): try: short = row.find_element("xpath", ".//td[1]").text name = row.find_element("xpath", ".//td[2]").text personsInGroup.append([short, name]) persons.add(short) #why??? except NoSuchElementException: print("NoSuchElementException") # use graphDB_Driver from neo4j to write a graphical databank with graphDB_Driver.session() as graphDB_Session: #1) create a node for each group with its name, use the DB-language "Cypher" graphDB_Session.run("""CREATE (:ADGROUP {name: $name})""", name=group) #2) insert name and short notation for short, name in personsInGroup: #reads out all short notations, already in DB with this name g = graphDB_Session.run("""MATCH (u:USER) WHERE u.displayName = $displayName RETURN u.short""", displayName=name).values() print(g) if g: #persons exists and therefore add the short notation to that name for item in g: if not item[0]: #attribute exists graphDB_Session.run("""MATCH (u:USER) WHERE u.displayName = $displayName SET u.short = $short""", displayName=name, short=short) else: #person does not exist # => create a node in user with its name and short notation #print(name, 'existiert nicht') graphDB_Session.run("""CREATE (u:USER {displayName: $displayName, short: $short})""", displayName=name, short=short) # Gruppenbeziehung anlegen graphDB_Session.run("MATCH (grp:ADGROUP {name: $name}), (u:USER {displayName: $displayName}) MERGE (u)-[:is_member]->(grp)", name=group, displayName=name) driver.close() return persons #find the teams each user is member of #next create nodes in DB for each team and connect it with persons if not done already def buildTeams(persons): driver = webdriver.Chrome() with graphDB_Driver.session() as graphDB_Session: for kuerzel in persons: #look on page of specific person driver.get("https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?user=" + kuerzel) field = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody/tr[7]/td[2]") #look at table "Member of following Teams:" for team in field.text.split('\n'): #have to do some string changes #if team does not exist create it or if just match it (select it) graphDB_Session.run("MERGE (t:TEAMS {name: $name})", name=team) #select each team and set the user to it, if not already existing graphDB_Session.run("MATCH (t:TEAMS {name: $name}), (u:USER {kuerzel: $kuerzel}) MERGE (u)-[:is_member]->(t)", name=team, kuerzel=kuerzel) driver.close() #did not look at rest yet. #mostly unimportant since special cases # Personen unter Sascha, die nicht in dem Teams Kanal WS_PPI_X sind #with graphDB_Driver.session() as graphDB_Session: # g = graphDB_Session.run("""MATCH (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER) # MATCH (ad:TEAMS {name: "WS_PPI_X"}) # WHERE not (u) -[:is_member]-> (ad) # CALL {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter} # RETURN vorgesetzter, u.jobTitle, u.displayName # ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values() #g # Personen unter Sascha, die nicht in der AD Gruppe prj_ppix sind #with graphDB_Driver.session() as graphDB_Session: # g = graphDB_Session.run("""match (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER) # match (ad:ADGROUP {name: "prj_ppix"}) # where not (u) -[:is_member]-> (ad) # call {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter} # RETURN vorgesetzter, u.jobTitle, u.displayName # ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values() #g if __name__ == "__main__": GroupsAtPpiX = findGroupNames() allPersons = getPersons(GroupsAtPpiX) print(allPersons) buildTeams(allPersons)