Skip to content
Snippets Groups Projects
Commit e39259d7 authored by markn92's avatar markn92
Browse files

async requests

parent fef34a7f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Pipfile
+ 2
− 3
View file @ e39259d7
......@@ -6,13 +6,12 @@ verify_ssl = true
[dev-packages]
pytest = "*"
pandas = "*"
xlrd = "*"
openpyxl = "*"
odf = "*"
[packages]
networkx = "*"
network2tikz = {editable = true,path = "/home/mark/Projekte/network2tikz"}
aiohttp = "*"
cchardet = "*"
[requires]
python_version = "3.7"
Pipfile.lock
+ 126
− 29
View file @ e39259d7
{
"_meta": {
"hash": {
"sha256": "c7d493ff39406747f47d2c4cfff4a9c1fce490dd1c4fa600d3e0b5793326a174"
"sha256": "fc86a7b43db104513782b90f61e1233a4f1d49415d6fab92cfc6b53a2af94753"
},
"pipfile-spec": 6,
"requires": {
......@@ -16,6 +16,80 @@
]
},
"default": {
"aiohttp": {
"hashes": [
"sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
"sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
"sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
"sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
"sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
"sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
"sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
"sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
"sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
"sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
"sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
"sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
],
"index": "pypi",
"version": "==3.6.2"
},
"async-timeout": {
"hashes": [
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
],
"version": "==3.0.1"
},
"attrs": {
"hashes": [
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
],
"version": "==19.3.0"
},
"cchardet": {
"hashes": [
"sha256:0f6e4e464e332da776b9c1a34e4e83b6301d38c2724efc93848c46ade66d02bb",
"sha256:217a7008bd399bdb61f6a0a2570acc5c3a9f96140e0a0d089b9e748c4d4e4c4e",
"sha256:27b0f23088873d1dd36d2c8a2e45c9167e312e1aac7e4baeb47f7428a2669638",
"sha256:2a958fb093f69ee5f16be7a1aee5122e07aff4350fa4dc9b953b87c34468e605",
"sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6",
"sha256:2c05b66b12f9ab0493c5ffb666036fd8c9004a9cc9d5a9264dc24738b50ab8c3",
"sha256:4096759825a130cb27a58ddf6d58e10abdd0127d29fbf53fde26df7ad879737b",
"sha256:40c199f9c0569ac479fae7c4e12d2e16fc1e8237836b928474fdd228b8d11477",
"sha256:4486f6e5bdf06f0081d13832f2a061d9e90597eb02093fda9d37e3985e3b2ef2",
"sha256:54d2653520237ebbd2928f2c0f2eb7c616ee2b5194d73d945060cd54a7846b64",
"sha256:5e38cfad9d3ca0f571c4352e9ca0f5ab718508f492a37d3236ae70810140e250",
"sha256:68409e00d75ff13dd7a192ec49559f5527ee8959a51a9f4dd7b168df972b4d44",
"sha256:79b0e113144c2ef0050bc9fe647c7657c5298f3012ecd8937d930b24ddd61404",
"sha256:7a2d98df461d3f36b403fdd8d7890c823ed05bd98eb074412ed56fbfedb94751",
"sha256:7bba1cbb4358dc9a2d2da00f4b38b159a5483d2f3b1d698a7c2cae518f955170",
"sha256:84d2ce838cf3c2fe7f0517941702d42f7e598e5173632ec47a113cd521669b98",
"sha256:8b1d02c99f6444c63336a76638741eaf4ac4005b454e3b8252a40074bf0d84a1",
"sha256:8f7ade2578b2326a0a554c03f60c8d079331220179a592e83e143c9556b7f5b2",
"sha256:953fe382304b19f5aa8fc2da4b092a3bb58a477d33af4def4b81abdce4c9288c",
"sha256:acc96b4a8f756af289fa90ffa67ddef57401d99131e51e71872e3609483941ce",
"sha256:af284494ea6c40f9613b4d939abe585eb9290cb92037eab66122c93190fcb338",
"sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3",
"sha256:ccb9f6f06265382028468b47e726f2d42539256fb498d1b0e473c39037b42b8a",
"sha256:cf134e1cfb0c53f08abb1ab9158a7e7f859c3ddb451d5fe535a2cc5f2958a688",
"sha256:dff9480d9b6260f59ad10e1cec5be13905be5da88a4a2bd5a5bd4d49c49c4a05",
"sha256:e27771798c8ad50df1375e762d59369354af94eb8ac21eca5bfd1eeef589f545",
"sha256:f245f045054e8d6dab2a0e366d3c74f3a47fb7dec2595ae2035b234b1a829c7a",
"sha256:f5c94994d876d8709847c3a92643309d716f43716580a2e5831262366a9ee8b6",
"sha256:fd16f57ce42a72397cd9fe38977fc809eb02172731cb354572f28a6d8e4cf322"
],
"index": "pypi",
"version": "==2.1.6"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"decorator": {
"hashes": [
"sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760",
......@@ -23,6 +97,35 @@
],
"version": "==4.4.2"
},
"idna": {
"hashes": [
"sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
"sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
],
"version": "==2.9"
},
"multidict": {
"hashes": [
"sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1",
"sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35",
"sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928",
"sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969",
"sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e",
"sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78",
"sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1",
"sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136",
"sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8",
"sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2",
"sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e",
"sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4",
"sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5",
"sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd",
"sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab",
"sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20",
"sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3"
],
"version": "==4.7.5"
},
"network2tikz": {
"editable": true,
"path": "/home/mark/Projekte/network2tikz"
......@@ -60,6 +163,28 @@
"sha256:fdee7540d12519865b423af411bd60ddb513d2eb2cd921149b732854995bbf8b"
],
"version": "==1.18.3"
},
"yarl": {
"hashes": [
"sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
"sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
"sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
"sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
"sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
"sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
"sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
"sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
"sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
"sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
"sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
"sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
"sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
"sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
"sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
"sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
"sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
],
"version": "==1.4.2"
}
},
"develop": {
......@@ -70,12 +195,6 @@
],
"version": "==19.3.0"
},
"et-xmlfile": {
"hashes": [
"sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b"
],
"version": "==1.0.1"
},
"importlib-metadata": {
"hashes": [
"sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
......@@ -84,13 +203,6 @@
"markers": "python_version < '3.8'",
"version": "==1.6.0"
},
"jdcal": {
"hashes": [
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
"sha256:472872e096eb8df219c23f2689fc336668bdb43d194094b5cc1707e1640acfc8"
],
"version": "==1.4.1"
},
"more-itertools": {
"hashes": [
"sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c",
......@@ -124,13 +236,6 @@
],
"version": "==1.18.3"
},
"openpyxl": {
"hashes": [
"sha256:547a9fc6aafcf44abe358b89ed4438d077e9d92e4f182c87e2dc294186dc4b64"
],
"index": "pypi",
"version": "==3.0.3"
},
"packaging": {
"hashes": [
"sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3",
......@@ -217,14 +322,6 @@
],
"version": "==0.1.9"
},
"xlrd": {
"hashes": [
"sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2",
"sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde"
],
"index": "pypi",
"version": "==1.2.0"
},
"zipp": {
"hashes": [
"sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
......
evrouting/osm.py
+ 97
− 90
View file @ e39259d7
......@@ -13,12 +13,62 @@ Added :
"""
import copy
import urllib.request
import xml.sax
from math import radians, cos, sin, asin, sqrt
from pathlib import Path
from collections import namedtuple
import networkx
import networkx as nx
import aiohttp
import asyncio
from evrouting.graph_tools import DISTANCE_KEY
OsrmConf = namedtuple('OsrmConf',
['server', 'port', 'version', 'profile'],
defaults=('v1', 'driving')
)
class CachedDistance:
def __init__(self, graph, symmetric=True):
self._cache = {}
self.graph = graph
self.symmetric = symmetric
def d(self, u, v):
raise NotImplemented
def __getitem__(self, item):
if self.symmetric:
item = sorted(item)
u, v = item
try:
return self._cache[u, v]
except KeyError:
d = self.d(u, v)
self._cache[u, v] = d
return d
class AsyncCachedOSRMDistance(CachedDistance):
def __init__(self,
graph,
session,
symmetric=False,
osrm_config: OsrmConf = OsrmConf(server='0.0.0.0', port=5000)
):
super().__init__(graph, symmetric)
self.session = session
self.query_url = query_url
async def d(self, u, v):
loc_u = (self.graph[u]['lat'], self.graph[u]['long'])
loc_v = (self.graph[v]['lat'], self.graph[v]['long'])
async with self.session.get(self.query_url('route', [loc_u, loc_v])) as resp:
return resp
def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True):
......@@ -33,7 +83,7 @@ def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True):
# haversine formula
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
c = 2 * asin(sqrt(a))
r = 6371 # Radius of the Earth in kilometers. Use 3956 for miles
if unit_m:
......@@ -41,62 +91,9 @@ def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True):
return c * r
def download_osm(left=-73.4244, bottom=45.4302, right=-73.4010, top=45.4466, proxy=False, proxyHost="10.0.4.2", proxyPort="3128", cache=False, cacheTempDir="/tmp/tmpOSM/", verbose=True):
""" Return a filehandle to the downloaded data from osm api."""
if cache:
# cached tile filename
cachedTileFilename = "osm_map_{:.8f}_{:.8f}_{:.8f}_{:.8f}.map".format(left, bottom, right, top)
if verbose:
print("Cached tile filename :", cachedTileFilename)
cacheTempDir = Path(cacheTempDir)
cacheTempDir.mkdir(parents=True, exist_ok=True) # Create cache path if not exists
osmFile = Path(cacheTempDir / cachedTileFilename).resolve() # Replace the relative cache folder path to absolute path
if osmFile.is_file():
# download from the cache folder
if verbose:
print("Tile loaded from the cache folder.")
with open(osmFile, mode='r') as f:
content = f.read()
return content
if proxy:
# configure the urllib request with the proxy
proxy_handler = urllib.request.ProxyHandler({'https': 'https://' + proxyHost + ":" + proxyPort, 'http': 'http://' + proxyHost + ":" + proxyPort})
opener = urllib.request.build_opener(proxy_handler)
urllib.request.install_opener(opener)
request = "http://api.openstreetmap.org/api/0.6/map?bbox=%f,%f,%f,%f" % (left, bottom, right, top)
if verbose:
print("Download the tile from osm web api ... in progress")
print("Request :", request)
fp = urllib.request.urlopen(request)
content = fp.read().decode('utf-8')
if verbose:
print("OSM Tile downloaded")
if cache:
if verbose:
print("Write osm tile in the cache")
with open(osmFile, 'w') as f:
f.write(content)
if osmFile.is_file():
if verbose:
print("OSM tile written in the cache")
return content
def read_osm(osm_xml_data, is_xml_string=True, only_roads=True):
def read_osm(osm_xml_data,
osrm_config: OsrmConf = OsrmConf(server='localhost', port=5000)
) -> nx.DiGraph:
"""Read graph in OSM format from file specified by name or by stream object.
Parameters
----------
......@@ -106,15 +103,16 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True):
-------
G : Graph
Examples
--------
>>> G=nx.read_osm(nx.download_osm(-122.33,47.60,-122.31,47.61))
>>> import matplotlib.pyplot as plt
>>> plt.plot([G.node[n]['lat']for n in G], [G.node[n]['lon'] for n in G], 'o', color='k')
>>> plt.show()
"""
osm = OSM(osm_xml_data, is_xml_string=is_xml_string)
G = networkx.DiGraph()
only_roads = osrm_config.profile == 'driving'
def query_url(service, coordinates):
return f'http://{osrm_config.server}:{osrm_config.port}' \
f'/{service}/{osrm_config.version}/{osrm_config.profile}/' \
f'{";".join([f"{lon},{lat}" for lat, lon in coordinates])}'
osm = OSM(osm_xml_data)
G = nx.DiGraph()
## Add ways
for w in osm.ways.values():
......@@ -124,15 +122,15 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True):
if ('oneway' in w.tags):
if (w.tags['oneway'] == 'yes'):
# ONLY ONE DIRECTION
networkx.add_path(G, w.nds, id=w.id)
nx.add_path(G, w.nds, id=w.id)
else:
# BOTH DIRECTION
networkx.add_path(G, w.nds, id=w.id)
networkx.add_path(G, w.nds[::-1], id=w.id)
nx.add_path(G, w.nds, id=w.id)
nx.add_path(G, w.nds[::-1], id=w.id)
else:
# BOTH DIRECTION
networkx.add_path(G, w.nds, id=w.id)
networkx.add_path(G, w.nds[::-1], id=w.id)
nx.add_path(G, w.nds, id=w.id)
nx.add_path(G, w.nds[::-1], id=w.id)
# Complete the used nodes' information
coordinates_map = {}
......@@ -143,14 +141,26 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True):
G.nodes[n_id]['id'] = n.id
coordinates_map[n_id] = (n.lon, n.lat)
# Estimate the length of each way
for u, v, d in G.edges(data=True):
distance = haversine_distance(G.nodes[u]['lon'], G.nodes[u]['lat'], G.nodes[v]['lon'], G.nodes[v]['lat'], unit_m=True) # Give a realistic distance estimation (neither EPSG nor projection nor reference system are specified)
asyncio.run(augment_distances(G, query_url))
G = nx.relabel_nodes(G, coordinates_map)
return G
G.add_weighted_edges_from([(u, v, distance)], weight='havlen')
G = networkx.relabel_nodes(G, coordinates_map)
return G
async def augment_distances(G, url_factory):
# Estimate the length of each way
async with aiohttp.ClientSession() as session:
for u, v, d in G.edges(data=True):
url = url_factory(
'route',
[
(G.nodes[u]['lat'], G.nodes[u]['lon']),
(G.nodes[v]['lat'], G.nodes[v]['lon'])
])
async with session.get(url) as resp:
resp.raise_for_status()
resp = await resp.json()
duration = resp['routes'][0]['duration']
G.add_weighted_edges_from([(u, v, duration)], weight=DISTANCE_KEY)
class Node(object):
......@@ -161,7 +171,7 @@ class Node(object):
self.tags = {}
def __str__(self):
return "Node (id : %s) lon : %s, lat : %s "%(self.id, self.lon, self.lat)
return "Node (id : %s) lon : %s, lat : %s " % (self.id, self.lon, self.lat)
class Way(object):
......@@ -174,14 +184,14 @@ class Way(object):
def split(self, dividers):
# slice the node-array using this nifty recursive function
def slice_array(ar, dividers):
for i in range(1,len(ar)-1):
if dividers[ar[i]]>1:
left = ar[:i+1]
for i in range(1, len(ar) - 1):
if dividers[ar[i]] > 1:
left = ar[:i + 1]
right = ar[i:]
rightsliced = slice_array(right, dividers)
return [left]+rightsliced
return [left] + rightsliced
return [ar]
slices = slice_array(self.nds, dividers)
......@@ -200,7 +210,7 @@ class Way(object):
class OSM(object):
def __init__(self, osm_xml_data, is_xml_string=True):
def __init__(self, osm_xml_data):
""" File can be either a filename or stream/file object.
set `is_xml_string=False` if osm_xml_data is a filename or a file stream.
......@@ -245,11 +255,8 @@ class OSM(object):
def characters(self, chars):
pass
if is_xml_string:
xml.sax.parse(osm_xml_data, OSMHandler)
else:
with open(osm_xml_data, mode='r') as f:
xml.sax.parse(f, OSMHandler)
with open(osm_xml_data, mode='r') as f:
xml.sax.parse(f, OSMHandler)
self.nodes = nodes
self.ways = ways
......
tests/osm/test_osm_charge.py
+ 9
− 0
View file @ e39259d7
import os
import networkx as nx
from evrouting.osm import read_osm
def test_read_osm():
G: nx.DiGraph = read_osm(os.path.join(os.path.dirname(__file__), 'static/map.osm'))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment