Upload files to "source/Dobby"
This commit is contained in:
286
source/Dobby/Dobby.py
Normal file
286
source/Dobby/Dobby.py
Normal file
@@ -0,0 +1,286 @@
|
|||||||
|
from selenium import webdriver
|
||||||
|
from selenium.common.exceptions import NoSuchElementException
|
||||||
|
from selenium.webdriver.support.ui import WebDriverWait
|
||||||
|
from selenium.webdriver.support import expected_conditions as EC
|
||||||
|
import numpy as np
|
||||||
|
import time
|
||||||
|
|
||||||
|
#some notes:
|
||||||
|
#the links to find elements can be found with:
|
||||||
|
#right click on browser and choose Untersuchen(Q)
|
||||||
|
#shows all elements => find the needed table by hand
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
define a driver = mousepointer using Chrome as Browser (Firefox did not work)
|
||||||
|
OUT:
|
||||||
|
driver: driver Selenium Object, which can connect to website
|
||||||
|
'''
|
||||||
|
def getDriver():
|
||||||
|
#definitely need sleep since first have to establish Selenium
|
||||||
|
time.sleep(10)
|
||||||
|
options = webdriver.ChromeOptions()
|
||||||
|
options.add_argument('--disable-blink-features=AutomationControlled')
|
||||||
|
options.add_argument('--ignore-ssl-errors=yes')
|
||||||
|
options.add_argument('--ignore-certificate-errors')
|
||||||
|
|
||||||
|
print("Now will connect")
|
||||||
|
driver = webdriver.Remote(command_executor='http://sel_browser_test_vimes:4444/wd/hub', options=options) #sel_browser_test gives the URL of that container, so that we always use correct URL to connect
|
||||||
|
#also need _vimes since we changed the name of the container in this dir
|
||||||
|
print("did connect successfully")
|
||||||
|
|
||||||
|
return driver
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
find names of each Group of PPI.X on Dobby
|
||||||
|
IN:
|
||||||
|
driver: driver Selenium Object, which can connect to website
|
||||||
|
OUT:
|
||||||
|
groupsPpiX: Array(n) of string
|
||||||
|
'''
|
||||||
|
def findGroupNames(driver):
|
||||||
|
groupsPpiX = []
|
||||||
|
|
||||||
|
#define the link to access and access it
|
||||||
|
pathDobbyProjects = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte"
|
||||||
|
print(pathDobbyProjects)
|
||||||
|
try:
|
||||||
|
driver.get(pathDobbyProjects)
|
||||||
|
print("on website now")
|
||||||
|
except:
|
||||||
|
print("could not connect to Dobby website")
|
||||||
|
|
||||||
|
#get data from Dobby
|
||||||
|
try:
|
||||||
|
#get the table with all projects (the lower table)
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody")
|
||||||
|
|
||||||
|
#go through each row in that table
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
groupId = row.find_element("xpath", ".//td[1]").text #first column in table is Group
|
||||||
|
print(groupId)
|
||||||
|
groupDesc = row.find_element("xpath", ".//td[2]").text #second is Description
|
||||||
|
if groupId[:8] == 'prj_ppix': #only choose groups starting like that = from PPI.X
|
||||||
|
groupsPpiX.append([groupId, groupDesc]) #append
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
#driver.close()
|
||||||
|
print("done going through all projects")
|
||||||
|
except:
|
||||||
|
print("no such tables found, where we get group names")
|
||||||
|
|
||||||
|
return groupsPpiX
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
find all persons working at PPI.X
|
||||||
|
and save names and short notation
|
||||||
|
IN:
|
||||||
|
groupsNames: Array(n,2) of string
|
||||||
|
OUT:
|
||||||
|
persons: set(m) of string arrays(2)
|
||||||
|
'''
|
||||||
|
def getPersons(driver, groupsNames):
|
||||||
|
persons = set()
|
||||||
|
groupPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?group="
|
||||||
|
|
||||||
|
for group, groupDesc in groupsNames:
|
||||||
|
try:
|
||||||
|
driver.get(groupPath + group)
|
||||||
|
try:
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody")
|
||||||
|
|
||||||
|
#go though each row in table with short notation and name
|
||||||
|
#do not consider first row (tr[not(position()=1)]) since caption only
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
short = row.find_element("xpath", ".//td[1]").text
|
||||||
|
print(short)
|
||||||
|
name = row.find_element("xpath", ".//td[2]").text
|
||||||
|
#append short notation to the set of all persons, but use tuple, because they are differentiable
|
||||||
|
persons.add(tuple([short, name]))
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
except:
|
||||||
|
print("no such tables found, where we get person names")
|
||||||
|
except:
|
||||||
|
print("could not connect to group webpage")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#driver.close()
|
||||||
|
return persons
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
scrappes all data from Dobby persons page
|
||||||
|
which only are their phone numbers
|
||||||
|
it is the same order as the names
|
||||||
|
IN:
|
||||||
|
driver: driver Selenium object
|
||||||
|
names: Array(n,2) of string
|
||||||
|
OUT:
|
||||||
|
allNumbers: array(n, 2) of string
|
||||||
|
'''
|
||||||
|
def getDobbyPersonData(driver, names):
|
||||||
|
personPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?user="
|
||||||
|
allNumbers = np.zeros((len(names), 2), dtype='U60')
|
||||||
|
|
||||||
|
for s, short in enumerate(names):
|
||||||
|
print(short[0]) #short[0] has kuerzel and short[1] the full name
|
||||||
|
print(personPath + short[0])
|
||||||
|
driver.get(personPath + short[0])
|
||||||
|
|
||||||
|
mobileNumber = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody/tr[9]/td[2]").text
|
||||||
|
print(mobileNumber)
|
||||||
|
#phoneNumber = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody/tr[8]/td[2]").text
|
||||||
|
allNumbers[s, 0] = mobileNumber
|
||||||
|
allNumbers[s, 1] = short[0]
|
||||||
|
print(allNumbers)
|
||||||
|
|
||||||
|
return allNumbers
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
From here on only some tests when debugging
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
test the Driver in Docker with a public webpage on YOUTUBE and print some data
|
||||||
|
also use the Proxy we need for Dobby
|
||||||
|
'''
|
||||||
|
def testDriver():
|
||||||
|
#need to wait a little to assure that the selenium container is up and running, before connecting to it
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
options = webdriver.ChromeOptions()
|
||||||
|
PROXY = "www-cache.ppi.int:3128" #PPI Proxy Name
|
||||||
|
options.add_argument('--disable-blink-features=AutomationControlled')
|
||||||
|
options.add_argument('--ignore-ssl-errors=yes')
|
||||||
|
options.add_argument('--ignore-certificate-errors')
|
||||||
|
#options.add_argument('--proxy-server=%s' % PROXY) #set Proxy to get onto dobby.int, but does not work
|
||||||
|
|
||||||
|
print("Now will connect")
|
||||||
|
driver = webdriver.Remote(command_executor='http://sel_browser_test_vimes:4444/wd/hub', options=options) #sel_browser_test gives the URL of that container, so that we always use correct URL to connect
|
||||||
|
#also need _vimes since we changed the name of the container in this dir
|
||||||
|
print("did connect successfully")
|
||||||
|
|
||||||
|
|
||||||
|
#just a test, to see if it works for a public website => it does
|
||||||
|
pathTest = "https://about.youtube/"
|
||||||
|
driver.get(pathTest)
|
||||||
|
print(pathTest)
|
||||||
|
print("someTesting only")
|
||||||
|
#print("full page")
|
||||||
|
#print(driver.page_source)
|
||||||
|
|
||||||
|
#driver will wait for up to 10 seconds
|
||||||
|
wait = WebDriverWait(driver, 10)
|
||||||
|
|
||||||
|
try:
|
||||||
|
elements = wait.until(EC.presence_of_all_elements_located(("xpath", '//*[@id="content"]')))
|
||||||
|
print(elements)
|
||||||
|
for e in elements:
|
||||||
|
print(e.get_attribute("class"))
|
||||||
|
except:
|
||||||
|
print("Youtube does not work")
|
||||||
|
|
||||||
|
try:
|
||||||
|
print("now the element of the wrong page")
|
||||||
|
elements = wait.until(EC.presence_of_all_elements_located(("id", 'sub-frame-error')))
|
||||||
|
print(elements)
|
||||||
|
except:
|
||||||
|
print("at least not the crashing proxy webpage")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
This was used to test the connection to the Dobby webpage
|
||||||
|
for example if the Proxy can help to connect
|
||||||
|
or how the crashed webpage looks like
|
||||||
|
'''
|
||||||
|
def testDobbyCrash():
|
||||||
|
time.sleep(5)
|
||||||
|
#define a driver = mousepointer using Chrome as Browser (Firefox did not work)
|
||||||
|
options = webdriver.ChromeOptions()
|
||||||
|
PROXY = "www-cache.ppi.int:3128" #PPI Proxy Name
|
||||||
|
options.add_argument('--disable-blink-features=AutomationControlled')
|
||||||
|
options.add_argument('--ignore-ssl-errors=yes')
|
||||||
|
options.add_argument('--ignore-certificate-errors')
|
||||||
|
#options.add_argument('--proxy-server=%s' % PROXY) #set Proxy to get onto dobby.int
|
||||||
|
|
||||||
|
print("Now will connect")
|
||||||
|
driver = webdriver.Remote(command_executor='http://sel_browser_test_vimes:4444/wd/hub', options=options) #sel_browser_test gives the URL of that container, so that we always use correct URL to connect
|
||||||
|
#also need _vimes since we changed the name of the container in this dir
|
||||||
|
print("did connect successfully")
|
||||||
|
|
||||||
|
|
||||||
|
#define the link to access and access it
|
||||||
|
pathDobbyProjects = "https://dobby.ppi.int"
|
||||||
|
print(pathDobbyProjects)
|
||||||
|
driver.get(pathDobbyProjects)
|
||||||
|
time.sleep(5)
|
||||||
|
print("full page")
|
||||||
|
print(driver.find_elements("xpath", "/html/body/img"))
|
||||||
|
#print(driver.page_source)
|
||||||
|
|
||||||
|
pathDobbyProjects = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte"
|
||||||
|
try:
|
||||||
|
driver.get(pathDobbyProjects)
|
||||||
|
print("on website now")
|
||||||
|
|
||||||
|
except:
|
||||||
|
print("could not connect to Dobby website")
|
||||||
|
|
||||||
|
time.sleep(5)
|
||||||
|
notFound = driver.find_elements("xpath", "/html/body/div/div[1]/div[1]/h1")
|
||||||
|
print("is website not found?")
|
||||||
|
print(notFound)
|
||||||
|
notFound = driver.find_elements("id", "root")
|
||||||
|
print("maybe root?")
|
||||||
|
print(notFound)
|
||||||
|
|
||||||
|
|
||||||
|
#find the table with all projects (the lower table)
|
||||||
|
table = driver.find_elements("xpath", "/html/body/div/div/table[2]/tbody/tr[3]")
|
||||||
|
print(table)
|
||||||
|
|
||||||
|
|
||||||
|
#driver will wait for up to 10 seconds
|
||||||
|
wait = WebDriverWait(driver, 10)
|
||||||
|
try:
|
||||||
|
print("now the element of the wrong page")
|
||||||
|
elements = wait.until(EC.presence_of_all_elements_located(("id", 'sub-frame-error')))
|
||||||
|
print(elements)
|
||||||
|
except:
|
||||||
|
print("at least not the crashing proxy webpage")
|
||||||
|
|
||||||
|
|
||||||
117
source/Dobby/DobbyBackup.py
Normal file
117
source/Dobby/DobbyBackup.py
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
from selenium import webdriver
|
||||||
|
from selenium.common.exceptions import NoSuchElementException
|
||||||
|
from neo4j import GraphDatabase
|
||||||
|
|
||||||
|
uri = "bolt://localhost:7687"
|
||||||
|
userName = "neo4j"
|
||||||
|
password = "org"
|
||||||
|
graphDB_Driver = GraphDatabase.driver(uri, auth=(userName, password))
|
||||||
|
|
||||||
|
|
||||||
|
projekte = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte"
|
||||||
|
|
||||||
|
driver = webdriver.Firefox()
|
||||||
|
driver.get(projekte)
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]")
|
||||||
|
|
||||||
|
groupsPpiX = []
|
||||||
|
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
groupId = row.find_element("xpath", ".//td[1]").text
|
||||||
|
groupDesc = row.find_element("xpath", ".//td[2]").text
|
||||||
|
if groupId[:8] == 'prj_ppix':
|
||||||
|
groupsPpiX.append([groupId, groupDesc])
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
|
||||||
|
driver = webdriver.Firefox()
|
||||||
|
|
||||||
|
personen = set()
|
||||||
|
|
||||||
|
for group, groupDesc in groupsPpiX:
|
||||||
|
driver.get("https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?group=" + group)
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody")
|
||||||
|
|
||||||
|
personenInGruppe = []
|
||||||
|
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
kuerzel = row.find_element("xpath", ".//td[1]").text
|
||||||
|
name = row.find_element("xpath", ".//td[2]").text
|
||||||
|
personenInGruppe.append([kuerzel, name])
|
||||||
|
personen.add(kuerzel)
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
#Gruppe anlegen
|
||||||
|
graphDB_Session.run("""CREATE (:ADGROUP {name: $name})""", name=group)
|
||||||
|
|
||||||
|
#Personen zur Gruppe verknüpfen
|
||||||
|
for kuerzel, name in personenInGruppe:
|
||||||
|
g = graphDB_Session.run("""MATCH (u:USER)
|
||||||
|
WHERE u.displayName = $displayName
|
||||||
|
RETURN u.kuerzel""", displayName=name).values()
|
||||||
|
if g:
|
||||||
|
#Person existiert
|
||||||
|
for item in g:
|
||||||
|
if not item[0]:
|
||||||
|
#Attribut existiert nicht
|
||||||
|
graphDB_Session.run("""MATCH (u:USER)
|
||||||
|
WHERE u.displayName = $displayName
|
||||||
|
SET u.kuerzel = $kuerzel""",
|
||||||
|
displayName=name,
|
||||||
|
kuerzel=kuerzel)
|
||||||
|
else:
|
||||||
|
#Person existiert nicht
|
||||||
|
#print(name, 'existiert nicht')
|
||||||
|
graphDB_Session.run("""CREATE (u:USER {displayName: $displayName,
|
||||||
|
kuerzel: $kuerzel})""",
|
||||||
|
displayName=name,
|
||||||
|
kuerzel=kuerzel)
|
||||||
|
# Gruppenbeziehung anlegen
|
||||||
|
graphDB_Session.run("MATCH (grp:ADGROUP {name: $name}), (u:USER {displayName: $displayName}) MERGE (u)-[:is_member]->(grp)",
|
||||||
|
name=group,
|
||||||
|
displayName=name)
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
driver = webdriver.Firefox()
|
||||||
|
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
for kuerzel in personen:
|
||||||
|
driver.get("https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?user=" + kuerzel)
|
||||||
|
field = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody/tr[7]/td[2]")
|
||||||
|
|
||||||
|
for team in field.text.split('\n'):
|
||||||
|
graphDB_Session.run("MERGE (t:TEAMS {name: $name})", name=team)
|
||||||
|
graphDB_Session.run("MATCH (t:TEAMS {name: $name}), (u:USER {kuerzel: $kuerzel}) MERGE (u)-[:is_member]->(t)",
|
||||||
|
name=team,
|
||||||
|
kuerzel=kuerzel)
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Personen unter Sascha, die nicht in dem Teams Kanal WS_PPI_X sind
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
g = graphDB_Session.run("""MATCH (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER)
|
||||||
|
MATCH (ad:TEAMS {name: "WS_PPI_X"})
|
||||||
|
WHERE not (u) -[:is_member]-> (ad)
|
||||||
|
CALL {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}
|
||||||
|
RETURN vorgesetzter, u.jobTitle, u.displayName
|
||||||
|
ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values()
|
||||||
|
g
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Personen unter Sascha, die nicht in der AD Gruppe prj_ppix sind
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
g = graphDB_Session.run("""match (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER)
|
||||||
|
match (ad:ADGROUP {name: "prj_ppix"})
|
||||||
|
where not (u) -[:is_member]-> (ad)
|
||||||
|
call {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}
|
||||||
|
RETURN vorgesetzter, u.jobTitle, u.displayName
|
||||||
|
ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values()
|
||||||
|
g
|
||||||
174
source/Dobby/DobbyGraphicalDB.py
Normal file
174
source/Dobby/DobbyGraphicalDB.py
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
from selenium import webdriver
|
||||||
|
from selenium.common.exceptions import NoSuchElementException
|
||||||
|
from neo4j import GraphDatabase
|
||||||
|
|
||||||
|
|
||||||
|
#some notes:
|
||||||
|
#the links to find elements can be found with:
|
||||||
|
#right click on browser and choose Untersuchen(Q)
|
||||||
|
#shows all elements => find the needed table by hand
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#define the driver for the databank
|
||||||
|
uri = "bolt://127.0.0.1:7687"
|
||||||
|
userName = "neo4j"
|
||||||
|
password = "org"
|
||||||
|
graphDB_Driver = GraphDatabase.driver(uri, auth=(userName, password))
|
||||||
|
|
||||||
|
|
||||||
|
#find names of each Group of PPI.X on Dobby
|
||||||
|
def findGroupNames():
|
||||||
|
groupsPpiX = []
|
||||||
|
|
||||||
|
#define a driver = mousepointer using Chrome as Browser (Firefox did not work)
|
||||||
|
driver = webdriver.Chrome()
|
||||||
|
#define the link to access and access it
|
||||||
|
projectPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte"
|
||||||
|
driver.get(projectPath)
|
||||||
|
#find the table with all projects (the lower table)
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]")
|
||||||
|
|
||||||
|
|
||||||
|
#go through each row in that table
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
groupId = row.find_element("xpath", ".//td[1]").text #first column in table if Group
|
||||||
|
groupDesc = row.find_element("xpath", ".//td[2]").text #second is Description
|
||||||
|
if groupId[:8] == 'prj_ppix': #only choose groups starting like that = from PPI.X
|
||||||
|
groupsPpiX.append([groupId, groupDesc])
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
print(groupsPpiX)
|
||||||
|
return groupsPpiX
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#find all persons working at PPI.X
|
||||||
|
#save names and short notation in
|
||||||
|
def getPersons(groupsNames):
|
||||||
|
persons = set()
|
||||||
|
driver = webdriver.Chrome()
|
||||||
|
groupPath = "https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?group="
|
||||||
|
|
||||||
|
for group, groupDesc in groupsNames:
|
||||||
|
driver.get(groupPath + group)
|
||||||
|
table = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody")
|
||||||
|
|
||||||
|
personsInGroup = [] #collect all persons (name+short notation) in that group we are locking at now
|
||||||
|
|
||||||
|
#go though each row in table with short notation and name
|
||||||
|
#do not consider first row (tr[not(position()=1)]) since caption only
|
||||||
|
for row in table.find_elements("xpath", ".//tr[not(position()=1)]"):
|
||||||
|
try:
|
||||||
|
short = row.find_element("xpath", ".//td[1]").text
|
||||||
|
name = row.find_element("xpath", ".//td[2]").text
|
||||||
|
personsInGroup.append([short, name])
|
||||||
|
persons.add(short) #why???
|
||||||
|
except NoSuchElementException:
|
||||||
|
print("NoSuchElementException")
|
||||||
|
|
||||||
|
# use graphDB_Driver from neo4j to write a graphical databank
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
#1) create a node for each group with its name, use the DB-language "Cypher"
|
||||||
|
graphDB_Session.run("""CREATE (:ADGROUP {name: $name})""", name=group)
|
||||||
|
#2) insert name and short notation
|
||||||
|
for short, name in personsInGroup:
|
||||||
|
#reads out all short notations, already in DB with this name
|
||||||
|
g = graphDB_Session.run("""MATCH (u:USER)
|
||||||
|
WHERE u.displayName = $displayName
|
||||||
|
RETURN u.short""", displayName=name).values()
|
||||||
|
print(g)
|
||||||
|
if g:
|
||||||
|
#persons exists and therefore add the short notation to that name
|
||||||
|
for item in g:
|
||||||
|
if not item[0]:
|
||||||
|
#attribute exists
|
||||||
|
graphDB_Session.run("""MATCH (u:USER)
|
||||||
|
WHERE u.displayName = $displayName
|
||||||
|
SET u.short = $short""",
|
||||||
|
displayName=name,
|
||||||
|
short=short)
|
||||||
|
else:
|
||||||
|
#person does not exist
|
||||||
|
# => create a node in user with its name and short notation
|
||||||
|
#print(name, 'existiert nicht')
|
||||||
|
graphDB_Session.run("""CREATE (u:USER {displayName: $displayName,
|
||||||
|
short: $short})""",
|
||||||
|
displayName=name,
|
||||||
|
short=short)
|
||||||
|
# Gruppenbeziehung anlegen
|
||||||
|
graphDB_Session.run("MATCH (grp:ADGROUP {name: $name}), (u:USER {displayName: $displayName}) MERGE (u)-[:is_member]->(grp)",
|
||||||
|
name=group,
|
||||||
|
displayName=name)
|
||||||
|
driver.close()
|
||||||
|
return persons
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#find the teams each user is member of
|
||||||
|
#next create nodes in DB for each team and connect it with persons if not done already
|
||||||
|
def buildTeams(persons):
|
||||||
|
driver = webdriver.Chrome()
|
||||||
|
with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
for kuerzel in persons:
|
||||||
|
#look on page of specific person
|
||||||
|
driver.get("https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?user=" + kuerzel)
|
||||||
|
field = driver.find_element("xpath", "/html/body/div/div/table[2]/tbody/tr[7]/td[2]") #look at table "Member of following Teams:"
|
||||||
|
|
||||||
|
for team in field.text.split('\n'): #have to do some string changes
|
||||||
|
#if team does not exist create it or if just match it (select it)
|
||||||
|
graphDB_Session.run("MERGE (t:TEAMS {name: $name})", name=team)
|
||||||
|
#select each team and set the user to it, if not already existing
|
||||||
|
graphDB_Session.run("MATCH (t:TEAMS {name: $name}), (u:USER {kuerzel: $kuerzel}) MERGE (u)-[:is_member]->(t)",
|
||||||
|
name=team,
|
||||||
|
kuerzel=kuerzel)
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#did not look at rest yet.
|
||||||
|
#mostly unimportant since special cases
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Personen unter Sascha, die nicht in dem Teams Kanal WS_PPI_X sind
|
||||||
|
#with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
# g = graphDB_Session.run("""MATCH (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER)
|
||||||
|
# MATCH (ad:TEAMS {name: "WS_PPI_X"})
|
||||||
|
# WHERE not (u) -[:is_member]-> (ad)
|
||||||
|
# CALL {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}
|
||||||
|
# RETURN vorgesetzter, u.jobTitle, u.displayName
|
||||||
|
# ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values()
|
||||||
|
#g
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Personen unter Sascha, die nicht in der AD Gruppe prj_ppix sind
|
||||||
|
#with graphDB_Driver.session() as graphDB_Session:
|
||||||
|
# g = graphDB_Session.run("""match (s:USER {displayName: "Sascha Däsler"}) -[:vorgesetzter*]-> (u:USER)
|
||||||
|
# match (ad:ADGROUP {name: "prj_ppix"})
|
||||||
|
# where not (u) -[:is_member]-> (ad)
|
||||||
|
# call {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}
|
||||||
|
# RETURN vorgesetzter, u.jobTitle, u.displayName
|
||||||
|
# ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName""").values()
|
||||||
|
#g
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
GroupsAtPpiX = findGroupNames()
|
||||||
|
allPersons = getPersons(GroupsAtPpiX)
|
||||||
|
print(allPersons)
|
||||||
|
buildTeams(allPersons)
|
||||||
185
source/Dobby/dobby.ipynb
Normal file
185
source/Dobby/dobby.ipynb
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from selenium import webdriver\n",
|
||||||
|
"from selenium.common.exceptions import NoSuchElementException\n",
|
||||||
|
"from neo4j import GraphDatabase\n",
|
||||||
|
"\n",
|
||||||
|
"uri = \"bolt://localhost:7687\"\n",
|
||||||
|
"userName = \"neo4j\"\n",
|
||||||
|
"password = \"org\"\n",
|
||||||
|
"graphDB_Driver = GraphDatabase.driver(uri, auth=(userName, password)) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"projekte = \"https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?ou=Projekte\"\n",
|
||||||
|
"\n",
|
||||||
|
"driver = webdriver.Firefox()\n",
|
||||||
|
"driver.get(projekte)\n",
|
||||||
|
"table = driver.find_element(\"xpath\", \"/html/body/div/div/table[2]\")\n",
|
||||||
|
"\n",
|
||||||
|
"groupsPpiX = []\n",
|
||||||
|
"\n",
|
||||||
|
"for row in table.find_elements(\"xpath\", \".//tr[not(position()=1)]\"):\n",
|
||||||
|
" try:\n",
|
||||||
|
" groupId = row.find_element(\"xpath\", \".//td[1]\").text\n",
|
||||||
|
" groupDesc = row.find_element(\"xpath\", \".//td[2]\").text\n",
|
||||||
|
" if groupId[:8] == 'prj_ppix':\n",
|
||||||
|
" groupsPpiX.append([groupId, groupDesc])\n",
|
||||||
|
" except NoSuchElementException:\n",
|
||||||
|
" print(\"NoSuchElementException\")\n",
|
||||||
|
"driver.close()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"driver = webdriver.Firefox()\n",
|
||||||
|
"\n",
|
||||||
|
"personen = set()\n",
|
||||||
|
"\n",
|
||||||
|
"for group, groupDesc in groupsPpiX:\n",
|
||||||
|
" driver.get(\"https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?group=\" + group)\n",
|
||||||
|
" table = driver.find_element(\"xpath\", \"/html/body/div/div/table[2]/tbody\")\n",
|
||||||
|
"\n",
|
||||||
|
" personenInGruppe = []\n",
|
||||||
|
"\n",
|
||||||
|
" for row in table.find_elements(\"xpath\", \".//tr[not(position()=1)]\"):\n",
|
||||||
|
" try:\n",
|
||||||
|
" kuerzel = row.find_element(\"xpath\", \".//td[1]\").text\n",
|
||||||
|
" name = row.find_element(\"xpath\", \".//td[2]\").text\n",
|
||||||
|
" personenInGruppe.append([kuerzel, name])\n",
|
||||||
|
" personen.add(kuerzel)\n",
|
||||||
|
" except NoSuchElementException:\n",
|
||||||
|
" print(\"NoSuchElementException\")\n",
|
||||||
|
"\n",
|
||||||
|
" with graphDB_Driver.session() as graphDB_Session:\n",
|
||||||
|
" #Gruppe anlegen\n",
|
||||||
|
" graphDB_Session.run(\"\"\"CREATE (:ADGROUP {name: $name})\"\"\", name=group)\n",
|
||||||
|
"\n",
|
||||||
|
" #Personen zur Gruppe verknüpfen\n",
|
||||||
|
" for kuerzel, name in personenInGruppe:\n",
|
||||||
|
" g = graphDB_Session.run(\"\"\"MATCH (u:USER)\n",
|
||||||
|
" WHERE u.displayName = $displayName\n",
|
||||||
|
" RETURN u.kuerzel\"\"\", displayName=name).values()\n",
|
||||||
|
" if g:\n",
|
||||||
|
" #Person existiert\n",
|
||||||
|
" for item in g:\n",
|
||||||
|
" if not item[0]:\n",
|
||||||
|
" #Attribut existiert nicht\n",
|
||||||
|
" graphDB_Session.run(\"\"\"MATCH (u:USER)\n",
|
||||||
|
" WHERE u.displayName = $displayName\n",
|
||||||
|
" SET u.kuerzel = $kuerzel\"\"\", \n",
|
||||||
|
" displayName=name,\n",
|
||||||
|
" kuerzel=kuerzel)\n",
|
||||||
|
" else:\n",
|
||||||
|
" #Person existiert nicht\n",
|
||||||
|
" #print(name, 'existiert nicht')\n",
|
||||||
|
" graphDB_Session.run(\"\"\"CREATE (u:USER {displayName: $displayName, \n",
|
||||||
|
" kuerzel: $kuerzel})\"\"\", \n",
|
||||||
|
" displayName=name,\n",
|
||||||
|
" kuerzel=kuerzel)\n",
|
||||||
|
" # Gruppenbeziehung anlegen\n",
|
||||||
|
" graphDB_Session.run(\"MATCH (grp:ADGROUP {name: $name}), (u:USER {displayName: $displayName}) MERGE (u)-[:is_member]->(grp)\",\n",
|
||||||
|
" name=group,\n",
|
||||||
|
" displayName=name)\n",
|
||||||
|
"driver.close()\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"driver = webdriver.Firefox()\n",
|
||||||
|
"\n",
|
||||||
|
"with graphDB_Driver.session() as graphDB_Session:\n",
|
||||||
|
" for kuerzel in personen:\n",
|
||||||
|
" driver.get(\"https://dobby.ppi.int/cgi-bin/ad/adgroups.pl?user=\" + kuerzel)\n",
|
||||||
|
" field = driver.find_element(\"xpath\", \"/html/body/div/div/table[2]/tbody/tr[7]/td[2]\")\n",
|
||||||
|
"\n",
|
||||||
|
" for team in field.text.split('\\n'):\n",
|
||||||
|
" graphDB_Session.run(\"MERGE (t:TEAMS {name: $name})\", name=team)\n",
|
||||||
|
" graphDB_Session.run(\"MATCH (t:TEAMS {name: $name}), (u:USER {kuerzel: $kuerzel}) MERGE (u)-[:is_member]->(t)\",\n",
|
||||||
|
" name=team,\n",
|
||||||
|
" kuerzel=kuerzel)\n",
|
||||||
|
"driver.close()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Personen unter Sascha, die nicht in dem Teams Kanal WS_PPI_X sind\n",
|
||||||
|
"with graphDB_Driver.session() as graphDB_Session:\n",
|
||||||
|
" g = graphDB_Session.run(\"\"\"MATCH (s:USER {displayName: \"Sascha Däsler\"}) -[:vorgesetzter*]-> (u:USER)\n",
|
||||||
|
" MATCH (ad:TEAMS {name: \"WS_PPI_X\"})\n",
|
||||||
|
" WHERE not (u) -[:is_member]-> (ad)\n",
|
||||||
|
" CALL {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}\n",
|
||||||
|
" RETURN vorgesetzter, u.jobTitle, u.displayName\n",
|
||||||
|
" ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName\"\"\").values()\n",
|
||||||
|
"g"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Personen unter Sascha, die nicht in der AD Gruppe prj_ppix sind\n",
|
||||||
|
"with graphDB_Driver.session() as graphDB_Session:\n",
|
||||||
|
" g = graphDB_Session.run(\"\"\"match (s:USER {displayName: \"Sascha Däsler\"}) -[:vorgesetzter*]-> (u:USER)\n",
|
||||||
|
" match (ad:ADGROUP {name: \"prj_ppix\"})\n",
|
||||||
|
" where not (u) -[:is_member]-> (ad)\n",
|
||||||
|
" call {with u match (v:USER) -[:vorgesetzter]-> (u) return v.displayName as vorgesetzter}\n",
|
||||||
|
" RETURN vorgesetzter, u.jobTitle, u.displayName\n",
|
||||||
|
" ORDER BY vorgesetzter, TOLOWER(u.jobTitle), u.displayName\"\"\").values()\n",
|
||||||
|
"g"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "venv",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.9"
|
||||||
|
},
|
||||||
|
"orig_nbformat": 4,
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "28676787f946bb7f76164fe7d563a7ae5bf108f75fcb12ae9589f51e6258b2b8"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user