Use pyquery-ql.py with files¶
Send a graphql query to GitHub and work with files for reports.
Supports Python 3.6+
In [1]:
import csv
import json
import os
import pprint
import requests
In [2]:
# get api token and set authorization
api_token = os.environ['GITHUB_API_TOKEN']
headers = {'Authorization': f'token {api_token}'}
In [3]:
# set url to a graphql endpoint
url = 'https://api.github.com/graphql'
In [4]:
# add a json query
query = """
{
organization(login: "jupyterhub") {
repositories(first: 30) {
nodes {
name
url
issues(states: OPEN) {
totalCount
}
pullRequests(states: OPEN) {
totalCount
}
}
}
}
}
"""
Make request and create json and csv files¶
In [5]:
# submit the request
r = requests.post(url=url, json={'query': query}, headers=headers)
In [6]:
# create a json file from response
with open('data.json', 'w') as f:
json.dump(r.json(), f)
In [7]:
# unpack the layers of json
nodes = r.json()['data']['organization']['repositories']['nodes']
unpacked = []
for node in nodes:
unpacked.append(node)
In [8]:
nodes
Out[8]:
[{'issues': {'totalCount': 141},
'name': 'jupyterhub',
'pullRequests': {'totalCount': 6},
'url': 'https://github.com/jupyterhub/jupyterhub'},
{'issues': {'totalCount': 11},
'name': 'configurable-http-proxy',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/configurable-http-proxy'},
{'issues': {'totalCount': 13},
'name': 'oauthenticator',
'pullRequests': {'totalCount': 2},
'url': 'https://github.com/jupyterhub/oauthenticator'},
{'issues': {'totalCount': 22},
'name': 'dockerspawner',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/dockerspawner'},
{'issues': {'totalCount': 2},
'name': 'sudospawner',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/sudospawner'},
{'issues': {'totalCount': 12},
'name': 'batchspawner',
'pullRequests': {'totalCount': 5},
'url': 'https://github.com/jupyterhub/batchspawner'},
{'issues': {'totalCount': 25},
'name': 'kubespawner',
'pullRequests': {'totalCount': 3},
'url': 'https://github.com/jupyterhub/kubespawner'},
{'issues': {'totalCount': 18},
'name': 'ldapauthenticator',
'pullRequests': {'totalCount': 2},
'url': 'https://github.com/jupyterhub/ldapauthenticator'},
{'issues': {'totalCount': 9},
'name': 'jupyterhub-deploy-docker',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/jupyterhub-deploy-docker'},
{'issues': {'totalCount': 13},
'name': 'jupyterhub-deploy-teaching',
'pullRequests': {'totalCount': 2},
'url': 'https://github.com/jupyterhub/jupyterhub-deploy-teaching'},
{'issues': {'totalCount': 5},
'name': 'jupyterhub-tutorial',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/jupyterhub-tutorial'},
{'issues': {'totalCount': 2},
'name': 'jupyterhub-deploy-hpc',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/jupyterhub-deploy-hpc'},
{'issues': {'totalCount': 4},
'name': 'systemdspawner',
'pullRequests': {'totalCount': 2},
'url': 'https://github.com/jupyterhub/systemdspawner'},
{'issues': {'totalCount': 5},
'name': 'wrapspawner',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/wrapspawner'},
{'issues': {'totalCount': 4},
'name': 'jupyterlab-hub',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/jupyterlab-hub'},
{'issues': {'totalCount': 2},
'name': 'nbserverproxy',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/nbserverproxy'},
{'issues': {'totalCount': 3},
'name': 'jupyterhub-example-kerberos',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/jupyterhub-example-kerberos'},
{'issues': {'totalCount': 10},
'name': 'hubshare',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/hubshare'},
{'issues': {'totalCount': 4},
'name': 'nbrsessionproxy',
'pullRequests': {'totalCount': 2},
'url': 'https://github.com/jupyterhub/nbrsessionproxy'},
{'issues': {'totalCount': 0},
'name': 'tmpauthenticator',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/tmpauthenticator'},
{'issues': {'totalCount': 89},
'name': 'zero-to-jupyterhub-k8s',
'pullRequests': {'totalCount': 3},
'url': 'https://github.com/jupyterhub/zero-to-jupyterhub-k8s'},
{'issues': {'totalCount': 3},
'name': 'helm-chart',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/helm-chart'},
{'issues': {'totalCount': 79},
'name': 'binderhub',
'pullRequests': {'totalCount': 7},
'url': 'https://github.com/jupyterhub/binderhub'},
{'issues': {'totalCount': 25},
'name': 'mybinder.org-deploy',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/mybinder.org-deploy'},
{'issues': {'totalCount': 9},
'name': 'binder',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/binder'},
{'issues': {'totalCount': 0},
'name': 'nullauthenticator',
'pullRequests': {'totalCount': 0},
'url': 'https://github.com/jupyterhub/nullauthenticator'},
{'issues': {'totalCount': 2},
'name': 'team-compass',
'pullRequests': {'totalCount': 1},
'url': 'https://github.com/jupyterhub/team-compass'}]
In [9]:
headers = ['name', 'url', 'issues', 'prs']
rows = []
for obj in unpacked:
new_dict = {'name':obj['name'], 'url':obj['url'], 'issues':obj['issues']['totalCount'], 'prs':obj['pullRequests']['totalCount']}
rows.append(new_dict)
In [10]:
with open('mydata.csv', 'w') as f:
f_csv = csv.DictWriter(f, headers)
f_csv.writeheader()
f_csv.writerows(rows)
Check file
In [11]:
%%bash
less mydata.csv
name,url,issues,prs
jupyterhub,https://github.com/jupyterhub/jupyterhub,141,6
configurable-http-proxy,https://github.com/jupyterhub/configurable-http-proxy,11,0
oauthenticator,https://github.com/jupyterhub/oauthenticator,13,2
dockerspawner,https://github.com/jupyterhub/dockerspawner,22,1
sudospawner,https://github.com/jupyterhub/sudospawner,2,0
batchspawner,https://github.com/jupyterhub/batchspawner,12,5
kubespawner,https://github.com/jupyterhub/kubespawner,25,3
ldapauthenticator,https://github.com/jupyterhub/ldapauthenticator,18,2
jupyterhub-deploy-docker,https://github.com/jupyterhub/jupyterhub-deploy-docker,9,1
jupyterhub-deploy-teaching,https://github.com/jupyterhub/jupyterhub-deploy-teaching,13,2
jupyterhub-tutorial,https://github.com/jupyterhub/jupyterhub-tutorial,5,0
jupyterhub-deploy-hpc,https://github.com/jupyterhub/jupyterhub-deploy-hpc,2,0
systemdspawner,https://github.com/jupyterhub/systemdspawner,4,2
wrapspawner,https://github.com/jupyterhub/wrapspawner,5,1
jupyterlab-hub,https://github.com/jupyterhub/jupyterlab-hub,4,0
nbserverproxy,https://github.com/jupyterhub/nbserverproxy,2,1
jupyterhub-example-kerberos,https://github.com/jupyterhub/jupyterhub-example-kerberos,3,0
hubshare,https://github.com/jupyterhub/hubshare,10,0
nbrsessionproxy,https://github.com/jupyterhub/nbrsessionproxy,4,2
tmpauthenticator,https://github.com/jupyterhub/tmpauthenticator,0,0
zero-to-jupyterhub-k8s,https://github.com/jupyterhub/zero-to-jupyterhub-k8s,89,3
helm-chart,https://github.com/jupyterhub/helm-chart,3,1
binderhub,https://github.com/jupyterhub/binderhub,79,7
mybinder.org-deploy,https://github.com/jupyterhub/mybinder.org-deploy,25,1
binder,https://github.com/jupyterhub/binder,9,1
nullauthenticator,https://github.com/jupyterhub/nullauthenticator,0,0
team-compass,https://github.com/jupyterhub/team-compass,2,1
Bring into pandas¶
In [12]:
import pandas as pd
In [13]:
df = pd.read_csv('mydata.csv')
In [14]:
df.columns
Out[14]:
Index(['name', 'url', 'issues', 'prs'], dtype='object')
In [15]:
# df.head()
Generate basic report of total open issues
In [16]:
# df.dtypes
In [17]:
# df.index
In [18]:
# df.values
Reports¶
In [19]:
# By repo name
df.sort_values(by=['name'])
Out[19]:
| name | url | issues | prs | |
|---|---|---|---|---|
| 5 | batchspawner | https://github.com/jupyterhub/batchspawner | 12 | 5 |
| 24 | binder | https://github.com/jupyterhub/binder | 9 | 1 |
| 22 | binderhub | https://github.com/jupyterhub/binderhub | 79 | 7 |
| 1 | configurable-http-proxy | https://github.com/jupyterhub/configurable-htt... | 11 | 0 |
| 3 | dockerspawner | https://github.com/jupyterhub/dockerspawner | 22 | 1 |
| 21 | helm-chart | https://github.com/jupyterhub/helm-chart | 3 | 1 |
| 17 | hubshare | https://github.com/jupyterhub/hubshare | 10 | 0 |
| 0 | jupyterhub | https://github.com/jupyterhub/jupyterhub | 141 | 6 |
| 8 | jupyterhub-deploy-docker | https://github.com/jupyterhub/jupyterhub-deplo... | 9 | 1 |
| 11 | jupyterhub-deploy-hpc | https://github.com/jupyterhub/jupyterhub-deplo... | 2 | 0 |
| 9 | jupyterhub-deploy-teaching | https://github.com/jupyterhub/jupyterhub-deplo... | 13 | 2 |
| 16 | jupyterhub-example-kerberos | https://github.com/jupyterhub/jupyterhub-examp... | 3 | 0 |
| 10 | jupyterhub-tutorial | https://github.com/jupyterhub/jupyterhub-tutorial | 5 | 0 |
| 14 | jupyterlab-hub | https://github.com/jupyterhub/jupyterlab-hub | 4 | 0 |
| 6 | kubespawner | https://github.com/jupyterhub/kubespawner | 25 | 3 |
| 7 | ldapauthenticator | https://github.com/jupyterhub/ldapauthenticator | 18 | 2 |
| 23 | mybinder.org-deploy | https://github.com/jupyterhub/mybinder.org-deploy | 25 | 1 |
| 18 | nbrsessionproxy | https://github.com/jupyterhub/nbrsessionproxy | 4 | 2 |
| 15 | nbserverproxy | https://github.com/jupyterhub/nbserverproxy | 2 | 1 |
| 25 | nullauthenticator | https://github.com/jupyterhub/nullauthenticator | 0 | 0 |
| 2 | oauthenticator | https://github.com/jupyterhub/oauthenticator | 13 | 2 |
| 4 | sudospawner | https://github.com/jupyterhub/sudospawner | 2 | 0 |
| 12 | systemdspawner | https://github.com/jupyterhub/systemdspawner | 4 | 2 |
| 26 | team-compass | https://github.com/jupyterhub/team-compass | 2 | 1 |
| 19 | tmpauthenticator | https://github.com/jupyterhub/tmpauthenticator | 0 | 0 |
| 13 | wrapspawner | https://github.com/jupyterhub/wrapspawner | 5 | 1 |
| 20 | zero-to-jupyterhub-k8s | https://github.com/jupyterhub/zero-to-jupyterh... | 89 | 3 |
In [20]:
# By open issue count
df.sort_values(by=['issues'], ascending=False)
Out[20]:
| name | url | issues | prs | |
|---|---|---|---|---|
| 0 | jupyterhub | https://github.com/jupyterhub/jupyterhub | 141 | 6 |
| 20 | zero-to-jupyterhub-k8s | https://github.com/jupyterhub/zero-to-jupyterh... | 89 | 3 |
| 22 | binderhub | https://github.com/jupyterhub/binderhub | 79 | 7 |
| 6 | kubespawner | https://github.com/jupyterhub/kubespawner | 25 | 3 |
| 23 | mybinder.org-deploy | https://github.com/jupyterhub/mybinder.org-deploy | 25 | 1 |
| 3 | dockerspawner | https://github.com/jupyterhub/dockerspawner | 22 | 1 |
| 7 | ldapauthenticator | https://github.com/jupyterhub/ldapauthenticator | 18 | 2 |
| 2 | oauthenticator | https://github.com/jupyterhub/oauthenticator | 13 | 2 |
| 9 | jupyterhub-deploy-teaching | https://github.com/jupyterhub/jupyterhub-deplo... | 13 | 2 |
| 5 | batchspawner | https://github.com/jupyterhub/batchspawner | 12 | 5 |
| 1 | configurable-http-proxy | https://github.com/jupyterhub/configurable-htt... | 11 | 0 |
| 17 | hubshare | https://github.com/jupyterhub/hubshare | 10 | 0 |
| 24 | binder | https://github.com/jupyterhub/binder | 9 | 1 |
| 8 | jupyterhub-deploy-docker | https://github.com/jupyterhub/jupyterhub-deplo... | 9 | 1 |
| 13 | wrapspawner | https://github.com/jupyterhub/wrapspawner | 5 | 1 |
| 10 | jupyterhub-tutorial | https://github.com/jupyterhub/jupyterhub-tutorial | 5 | 0 |
| 14 | jupyterlab-hub | https://github.com/jupyterhub/jupyterlab-hub | 4 | 0 |
| 18 | nbrsessionproxy | https://github.com/jupyterhub/nbrsessionproxy | 4 | 2 |
| 12 | systemdspawner | https://github.com/jupyterhub/systemdspawner | 4 | 2 |
| 16 | jupyterhub-example-kerberos | https://github.com/jupyterhub/jupyterhub-examp... | 3 | 0 |
| 21 | helm-chart | https://github.com/jupyterhub/helm-chart | 3 | 1 |
| 15 | nbserverproxy | https://github.com/jupyterhub/nbserverproxy | 2 | 1 |
| 11 | jupyterhub-deploy-hpc | https://github.com/jupyterhub/jupyterhub-deplo... | 2 | 0 |
| 4 | sudospawner | https://github.com/jupyterhub/sudospawner | 2 | 0 |
| 26 | team-compass | https://github.com/jupyterhub/team-compass | 2 | 1 |
| 19 | tmpauthenticator | https://github.com/jupyterhub/tmpauthenticator | 0 | 0 |
| 25 | nullauthenticator | https://github.com/jupyterhub/nullauthenticator | 0 | 0 |
In [21]:
# by open pr count
df.sort_values(by=['prs'], ascending=False)
Out[21]:
| name | url | issues | prs | |
|---|---|---|---|---|
| 22 | binderhub | https://github.com/jupyterhub/binderhub | 79 | 7 |
| 0 | jupyterhub | https://github.com/jupyterhub/jupyterhub | 141 | 6 |
| 5 | batchspawner | https://github.com/jupyterhub/batchspawner | 12 | 5 |
| 6 | kubespawner | https://github.com/jupyterhub/kubespawner | 25 | 3 |
| 20 | zero-to-jupyterhub-k8s | https://github.com/jupyterhub/zero-to-jupyterh... | 89 | 3 |
| 2 | oauthenticator | https://github.com/jupyterhub/oauthenticator | 13 | 2 |
| 7 | ldapauthenticator | https://github.com/jupyterhub/ldapauthenticator | 18 | 2 |
| 9 | jupyterhub-deploy-teaching | https://github.com/jupyterhub/jupyterhub-deplo... | 13 | 2 |
| 12 | systemdspawner | https://github.com/jupyterhub/systemdspawner | 4 | 2 |
| 18 | nbrsessionproxy | https://github.com/jupyterhub/nbrsessionproxy | 4 | 2 |
| 15 | nbserverproxy | https://github.com/jupyterhub/nbserverproxy | 2 | 1 |
| 24 | binder | https://github.com/jupyterhub/binder | 9 | 1 |
| 23 | mybinder.org-deploy | https://github.com/jupyterhub/mybinder.org-deploy | 25 | 1 |
| 21 | helm-chart | https://github.com/jupyterhub/helm-chart | 3 | 1 |
| 13 | wrapspawner | https://github.com/jupyterhub/wrapspawner | 5 | 1 |
| 8 | jupyterhub-deploy-docker | https://github.com/jupyterhub/jupyterhub-deplo... | 9 | 1 |
| 3 | dockerspawner | https://github.com/jupyterhub/dockerspawner | 22 | 1 |
| 26 | team-compass | https://github.com/jupyterhub/team-compass | 2 | 1 |
| 14 | jupyterlab-hub | https://github.com/jupyterhub/jupyterlab-hub | 4 | 0 |
| 16 | jupyterhub-example-kerberos | https://github.com/jupyterhub/jupyterhub-examp... | 3 | 0 |
| 17 | hubshare | https://github.com/jupyterhub/hubshare | 10 | 0 |
| 1 | configurable-http-proxy | https://github.com/jupyterhub/configurable-htt... | 11 | 0 |
| 19 | tmpauthenticator | https://github.com/jupyterhub/tmpauthenticator | 0 | 0 |
| 11 | jupyterhub-deploy-hpc | https://github.com/jupyterhub/jupyterhub-deplo... | 2 | 0 |
| 10 | jupyterhub-tutorial | https://github.com/jupyterhub/jupyterhub-tutorial | 5 | 0 |
| 4 | sudospawner | https://github.com/jupyterhub/sudospawner | 2 | 0 |
| 25 | nullauthenticator | https://github.com/jupyterhub/nullauthenticator | 0 | 0 |
In [22]:
# output data to a csv
# df.to_csv('issue_report.csv')