From e39259d75adeb694f318beaa76c7eed099b1df39 Mon Sep 17 00:00:00 2001 From: "niehues.mark@gmail.com" <niehues.mark@gmail.com> Date: Wed, 22 Apr 2020 14:16:13 +0200 Subject: [PATCH] async requests --- Pipfile | 5 +- Pipfile.lock | 155 +++++++++++++++++++++++------ evrouting/osm.py | 187 ++++++++++++++++++----------------- tests/osm/test_osm_charge.py | 9 ++ 4 files changed, 234 insertions(+), 122 deletions(-) diff --git a/Pipfile b/Pipfile index fc05385..396fbfe 100644 --- a/Pipfile +++ b/Pipfile @@ -6,13 +6,12 @@ verify_ssl = true [dev-packages] pytest = "*" pandas = "*" -xlrd = "*" -openpyxl = "*" -odf = "*" [packages] networkx = "*" network2tikz = {editable = true,path = "/home/mark/Projekte/network2tikz"} +aiohttp = "*" +cchardet = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 498c85c..c5f3371 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c7d493ff39406747f47d2c4cfff4a9c1fce490dd1c4fa600d3e0b5793326a174" + "sha256": "fc86a7b43db104513782b90f61e1233a4f1d49415d6fab92cfc6b53a2af94753" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,80 @@ ] }, "default": { + "aiohttp": { + "hashes": [ + "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e", + "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326", + "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a", + "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654", + "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a", + "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4", + "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17", + "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec", + "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd", + "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48", + "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59", + "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965" + ], + "index": "pypi", + "version": "==3.6.2" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "cchardet": { + "hashes": [ + "sha256:0f6e4e464e332da776b9c1a34e4e83b6301d38c2724efc93848c46ade66d02bb", + "sha256:217a7008bd399bdb61f6a0a2570acc5c3a9f96140e0a0d089b9e748c4d4e4c4e", + "sha256:27b0f23088873d1dd36d2c8a2e45c9167e312e1aac7e4baeb47f7428a2669638", + "sha256:2a958fb093f69ee5f16be7a1aee5122e07aff4350fa4dc9b953b87c34468e605", + "sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6", + "sha256:2c05b66b12f9ab0493c5ffb666036fd8c9004a9cc9d5a9264dc24738b50ab8c3", + "sha256:4096759825a130cb27a58ddf6d58e10abdd0127d29fbf53fde26df7ad879737b", + "sha256:40c199f9c0569ac479fae7c4e12d2e16fc1e8237836b928474fdd228b8d11477", + "sha256:4486f6e5bdf06f0081d13832f2a061d9e90597eb02093fda9d37e3985e3b2ef2", + "sha256:54d2653520237ebbd2928f2c0f2eb7c616ee2b5194d73d945060cd54a7846b64", + "sha256:5e38cfad9d3ca0f571c4352e9ca0f5ab718508f492a37d3236ae70810140e250", + "sha256:68409e00d75ff13dd7a192ec49559f5527ee8959a51a9f4dd7b168df972b4d44", + "sha256:79b0e113144c2ef0050bc9fe647c7657c5298f3012ecd8937d930b24ddd61404", + "sha256:7a2d98df461d3f36b403fdd8d7890c823ed05bd98eb074412ed56fbfedb94751", + "sha256:7bba1cbb4358dc9a2d2da00f4b38b159a5483d2f3b1d698a7c2cae518f955170", + "sha256:84d2ce838cf3c2fe7f0517941702d42f7e598e5173632ec47a113cd521669b98", + "sha256:8b1d02c99f6444c63336a76638741eaf4ac4005b454e3b8252a40074bf0d84a1", + "sha256:8f7ade2578b2326a0a554c03f60c8d079331220179a592e83e143c9556b7f5b2", + "sha256:953fe382304b19f5aa8fc2da4b092a3bb58a477d33af4def4b81abdce4c9288c", + "sha256:acc96b4a8f756af289fa90ffa67ddef57401d99131e51e71872e3609483941ce", + "sha256:af284494ea6c40f9613b4d939abe585eb9290cb92037eab66122c93190fcb338", + "sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3", + "sha256:ccb9f6f06265382028468b47e726f2d42539256fb498d1b0e473c39037b42b8a", + "sha256:cf134e1cfb0c53f08abb1ab9158a7e7f859c3ddb451d5fe535a2cc5f2958a688", + "sha256:dff9480d9b6260f59ad10e1cec5be13905be5da88a4a2bd5a5bd4d49c49c4a05", + "sha256:e27771798c8ad50df1375e762d59369354af94eb8ac21eca5bfd1eeef589f545", + "sha256:f245f045054e8d6dab2a0e366d3c74f3a47fb7dec2595ae2035b234b1a829c7a", + "sha256:f5c94994d876d8709847c3a92643309d716f43716580a2e5831262366a9ee8b6", + "sha256:fd16f57ce42a72397cd9fe38977fc809eb02172731cb354572f28a6d8e4cf322" + ], + "index": "pypi", + "version": "==2.1.6" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, "decorator": { "hashes": [ "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", @@ -23,6 +97,35 @@ ], "version": "==4.4.2" }, + "idna": { + "hashes": [ + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + ], + "version": "==2.9" + }, + "multidict": { + "hashes": [ + "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", + "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", + "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", + "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", + "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", + "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", + "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", + "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", + "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", + "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", + "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", + "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", + "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", + "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", + "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", + "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", + "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" + ], + "version": "==4.7.5" + }, "network2tikz": { "editable": true, "path": "/home/mark/Projekte/network2tikz" @@ -60,6 +163,28 @@ "sha256:fdee7540d12519865b423af411bd60ddb513d2eb2cd921149b732854995bbf8b" ], "version": "==1.18.3" + }, + "yarl": { + "hashes": [ + "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", + "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", + "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", + "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", + "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", + "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", + "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", + "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", + "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", + "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", + "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", + "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", + "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", + "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", + "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", + "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", + "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" + ], + "version": "==1.4.2" } }, "develop": { @@ -70,12 +195,6 @@ ], "version": "==19.3.0" }, - "et-xmlfile": { - "hashes": [ - "sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b" - ], - "version": "==1.0.1" - }, "importlib-metadata": { "hashes": [ "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", @@ -84,13 +203,6 @@ "markers": "python_version < '3.8'", "version": "==1.6.0" }, - "jdcal": { - "hashes": [ - "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba", - "sha256:472872e096eb8df219c23f2689fc336668bdb43d194094b5cc1707e1640acfc8" - ], - "version": "==1.4.1" - }, "more-itertools": { "hashes": [ "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", @@ -124,13 +236,6 @@ ], "version": "==1.18.3" }, - "openpyxl": { - "hashes": [ - "sha256:547a9fc6aafcf44abe358b89ed4438d077e9d92e4f182c87e2dc294186dc4b64" - ], - "index": "pypi", - "version": "==3.0.3" - }, "packaging": { "hashes": [ "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", @@ -217,14 +322,6 @@ ], "version": "==0.1.9" }, - "xlrd": { - "hashes": [ - "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", - "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" - ], - "index": "pypi", - "version": "==1.2.0" - }, "zipp": { "hashes": [ "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", diff --git a/evrouting/osm.py b/evrouting/osm.py index ded27a5..19d8d6d 100644 --- a/evrouting/osm.py +++ b/evrouting/osm.py @@ -13,12 +13,62 @@ Added : """ import copy -import urllib.request import xml.sax from math import radians, cos, sin, asin, sqrt -from pathlib import Path +from collections import namedtuple -import networkx +import networkx as nx +import aiohttp +import asyncio + +from evrouting.graph_tools import DISTANCE_KEY + +OsrmConf = namedtuple('OsrmConf', + ['server', 'port', 'version', 'profile'], + defaults=('v1', 'driving') + ) + + +class CachedDistance: + def __init__(self, graph, symmetric=True): + self._cache = {} + self.graph = graph + self.symmetric = symmetric + + def d(self, u, v): + raise NotImplemented + + def __getitem__(self, item): + if self.symmetric: + item = sorted(item) + u, v = item + + try: + return self._cache[u, v] + except KeyError: + d = self.d(u, v) + self._cache[u, v] = d + return d + + +class AsyncCachedOSRMDistance(CachedDistance): + def __init__(self, + graph, + session, + symmetric=False, + osrm_config: OsrmConf = OsrmConf(server='0.0.0.0', port=5000) + ): + super().__init__(graph, symmetric) + self.session = session + + self.query_url = query_url + + async def d(self, u, v): + loc_u = (self.graph[u]['lat'], self.graph[u]['long']) + loc_v = (self.graph[v]['lat'], self.graph[v]['long']) + + async with self.session.get(self.query_url('route', [loc_u, loc_v])) as resp: + return resp def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True): @@ -33,7 +83,7 @@ def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True): # haversine formula dlon = lon2 - lon1 dlat = lat2 - lat1 - a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2 + a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2 c = 2 * asin(sqrt(a)) r = 6371 # Radius of the Earth in kilometers. Use 3956 for miles if unit_m: @@ -41,62 +91,9 @@ def haversine_distance(lon1, lat1, lon2, lat2, unit_m=True): return c * r -def download_osm(left=-73.4244, bottom=45.4302, right=-73.4010, top=45.4466, proxy=False, proxyHost="10.0.4.2", proxyPort="3128", cache=False, cacheTempDir="/tmp/tmpOSM/", verbose=True): - """ Return a filehandle to the downloaded data from osm api.""" - if cache: - # cached tile filename - cachedTileFilename = "osm_map_{:.8f}_{:.8f}_{:.8f}_{:.8f}.map".format(left, bottom, right, top) - - if verbose: - print("Cached tile filename :", cachedTileFilename) - - cacheTempDir = Path(cacheTempDir) - cacheTempDir.mkdir(parents=True, exist_ok=True) # Create cache path if not exists - - osmFile = Path(cacheTempDir / cachedTileFilename).resolve() # Replace the relative cache folder path to absolute path - - if osmFile.is_file(): - # download from the cache folder - if verbose: - print("Tile loaded from the cache folder.") - - with open(osmFile, mode='r') as f: - content = f.read() - return content - - if proxy: - # configure the urllib request with the proxy - proxy_handler = urllib.request.ProxyHandler({'https': 'https://' + proxyHost + ":" + proxyPort, 'http': 'http://' + proxyHost + ":" + proxyPort}) - opener = urllib.request.build_opener(proxy_handler) - urllib.request.install_opener(opener) - - request = "http://api.openstreetmap.org/api/0.6/map?bbox=%f,%f,%f,%f" % (left, bottom, right, top) - - if verbose: - print("Download the tile from osm web api ... in progress") - print("Request :", request) - - fp = urllib.request.urlopen(request) - content = fp.read().decode('utf-8') - - if verbose: - print("OSM Tile downloaded") - - if cache: - if verbose: - print("Write osm tile in the cache") - - with open(osmFile, 'w') as f: - f.write(content) - - if osmFile.is_file(): - if verbose: - print("OSM tile written in the cache") - - return content - - -def read_osm(osm_xml_data, is_xml_string=True, only_roads=True): +def read_osm(osm_xml_data, + osrm_config: OsrmConf = OsrmConf(server='localhost', port=5000) + ) -> nx.DiGraph: """Read graph in OSM format from file specified by name or by stream object. Parameters ---------- @@ -106,15 +103,16 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True): ------- G : Graph - Examples - -------- - >>> G=nx.read_osm(nx.download_osm(-122.33,47.60,-122.31,47.61)) - >>> import matplotlib.pyplot as plt - >>> plt.plot([G.node[n]['lat']for n in G], [G.node[n]['lon'] for n in G], 'o', color='k') - >>> plt.show() """ - osm = OSM(osm_xml_data, is_xml_string=is_xml_string) - G = networkx.DiGraph() + only_roads = osrm_config.profile == 'driving' + + def query_url(service, coordinates): + return f'http://{osrm_config.server}:{osrm_config.port}' \ + f'/{service}/{osrm_config.version}/{osrm_config.profile}/' \ + f'{";".join([f"{lon},{lat}" for lat, lon in coordinates])}' + + osm = OSM(osm_xml_data) + G = nx.DiGraph() ## Add ways for w in osm.ways.values(): @@ -124,15 +122,15 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True): if ('oneway' in w.tags): if (w.tags['oneway'] == 'yes'): # ONLY ONE DIRECTION - networkx.add_path(G, w.nds, id=w.id) + nx.add_path(G, w.nds, id=w.id) else: # BOTH DIRECTION - networkx.add_path(G, w.nds, id=w.id) - networkx.add_path(G, w.nds[::-1], id=w.id) + nx.add_path(G, w.nds, id=w.id) + nx.add_path(G, w.nds[::-1], id=w.id) else: # BOTH DIRECTION - networkx.add_path(G, w.nds, id=w.id) - networkx.add_path(G, w.nds[::-1], id=w.id) + nx.add_path(G, w.nds, id=w.id) + nx.add_path(G, w.nds[::-1], id=w.id) # Complete the used nodes' information coordinates_map = {} @@ -143,14 +141,26 @@ def read_osm(osm_xml_data, is_xml_string=True, only_roads=True): G.nodes[n_id]['id'] = n.id coordinates_map[n_id] = (n.lon, n.lat) - # Estimate the length of each way - for u, v, d in G.edges(data=True): - distance = haversine_distance(G.nodes[u]['lon'], G.nodes[u]['lat'], G.nodes[v]['lon'], G.nodes[v]['lat'], unit_m=True) # Give a realistic distance estimation (neither EPSG nor projection nor reference system are specified) + asyncio.run(augment_distances(G, query_url)) + G = nx.relabel_nodes(G, coordinates_map) + return G - G.add_weighted_edges_from([(u, v, distance)], weight='havlen') - G = networkx.relabel_nodes(G, coordinates_map) - return G +async def augment_distances(G, url_factory): + # Estimate the length of each way + async with aiohttp.ClientSession() as session: + for u, v, d in G.edges(data=True): + url = url_factory( + 'route', + [ + (G.nodes[u]['lat'], G.nodes[u]['lon']), + (G.nodes[v]['lat'], G.nodes[v]['lon']) + ]) + async with session.get(url) as resp: + resp.raise_for_status() + resp = await resp.json() + duration = resp['routes'][0]['duration'] + G.add_weighted_edges_from([(u, v, duration)], weight=DISTANCE_KEY) class Node(object): @@ -161,7 +171,7 @@ class Node(object): self.tags = {} def __str__(self): - return "Node (id : %s) lon : %s, lat : %s "%(self.id, self.lon, self.lat) + return "Node (id : %s) lon : %s, lat : %s " % (self.id, self.lon, self.lat) class Way(object): @@ -174,14 +184,14 @@ class Way(object): def split(self, dividers): # slice the node-array using this nifty recursive function def slice_array(ar, dividers): - for i in range(1,len(ar)-1): - if dividers[ar[i]]>1: - left = ar[:i+1] + for i in range(1, len(ar) - 1): + if dividers[ar[i]] > 1: + left = ar[:i + 1] right = ar[i:] rightsliced = slice_array(right, dividers) - return [left]+rightsliced + return [left] + rightsliced return [ar] slices = slice_array(self.nds, dividers) @@ -200,7 +210,7 @@ class Way(object): class OSM(object): - def __init__(self, osm_xml_data, is_xml_string=True): + def __init__(self, osm_xml_data): """ File can be either a filename or stream/file object. set `is_xml_string=False` if osm_xml_data is a filename or a file stream. @@ -245,11 +255,8 @@ class OSM(object): def characters(self, chars): pass - if is_xml_string: - xml.sax.parse(osm_xml_data, OSMHandler) - else: - with open(osm_xml_data, mode='r') as f: - xml.sax.parse(f, OSMHandler) + with open(osm_xml_data, mode='r') as f: + xml.sax.parse(f, OSMHandler) self.nodes = nodes self.ways = ways diff --git a/tests/osm/test_osm_charge.py b/tests/osm/test_osm_charge.py index e69de29..92e2eb4 100644 --- a/tests/osm/test_osm_charge.py +++ b/tests/osm/test_osm_charge.py @@ -0,0 +1,9 @@ +import os + +import networkx as nx + +from evrouting.osm import read_osm + + +def test_read_osm(): + G: nx.DiGraph = read_osm(os.path.join(os.path.dirname(__file__), 'static/map.osm')) -- GitLab