diff --git a/migrations/2018-09-05-065252_places_osm/down.sql b/migrations/2018-09-05-065252_places_osm/down.sql index e1faf25..4ed4b88 100644 --- a/migrations/2018-09-05-065252_places_osm/down.sql +++ b/migrations/2018-09-05-065252_places_osm/down.sql @@ -1,2 +1,4 @@ ALTER TABLE places DROP COLUMN osm_id; ALTER TABLE places DROP COLUMN osm_level; + +CREATE UNIQUE INDEX places_name_idx ON places (place_name); diff --git a/migrations/2018-09-05-065252_places_osm/up.sql b/migrations/2018-09-05-065252_places_osm/up.sql index 3b82a2e..2d7eae9 100644 --- a/migrations/2018-09-05-065252_places_osm/up.sql +++ b/migrations/2018-09-05-065252_places_osm/up.sql @@ -3,4 +3,7 @@ ALTER TABLE places ADD COLUMN osm_id BIGINT UNIQUE; ALTER TABLE places ADD COLUMN osm_level SMALLINT; CREATE INDEX places_osm_idx ON places (osm_id); +CREATE INDEX places_osml_idx ON places (osm_level); +DROP INDEX places_name_idx; +CREATE UNIQUE INDEX places_name_idx ON places (place_name, osm_level); diff --git a/src/fetch_places.rs b/src/fetch_places.rs new file mode 100644 index 0000000..a99bf48 --- /dev/null +++ b/src/fetch_places.rs @@ -0,0 +1,163 @@ +use diesel; +use diesel::prelude::*; +use models::Coord; +use reqwest::Client; +use rustc_serialize::json::Json; +use slug::slugify; + +pub fn update_image_places( + c: &PgConnection, + image: i32, +) -> Result<(), String> { + use schema::positions::dsl::*; + let coord = match positions + .filter(photo_id.eq(image)) + .select((latitude, longitude)) + .first::<(i32, i32)>(c) + { + Ok((tlat, tlong)) => Coord { + x: f64::from(tlat) / 1e6, + y: f64::from(tlong) / 1e6, + }, + Err(diesel::NotFound) => { + return Err(format!( + "Image #{} does not exist or has no position", + image + )); + } + Err(err) => { + return Err(format!("Failed to get image position: {}", err)); + } + }; + debug!("Should get places for {:?}", coord); + let client = Client::new(); + match client + .post("https://overpass.kumi.systems/api/interpreter") + .body(format!( + "[out:json];is_in({},{});area._[admin_level];out;", + coord.x, coord.y, + )).send() + { + Ok(mut response) => { + if response.status().is_success() { + let data = Json::from_reader(&mut response).unwrap(); + let obj = data.as_object().unwrap(); + if let Some(elements) = + obj.get("elements").and_then(|o| o.as_array()) + { + for obj in elements { + if let (Some(t_osm_id), Some((name, level))) = + (osm_id(obj), name_and_level(obj)) + { + debug!("{}: {} (level {})", t_osm_id, name, level); + let place = { + use models::Place; + use schema::places::dsl::*; + places + .filter( + osm_id.eq(Some(t_osm_id)).or( + place_name + .eq(name) + .and(osm_id.is_null()), + ), + ).first::(c) + .or_else(|_| { + diesel::insert_into(places) + .values(( + place_name.eq(&name), + slug.eq(&slugify(&name)), + osm_id.eq(Some(t_osm_id)), + osm_level.eq(Some(level)), + )).get_result::(c) + .or_else(|_| { + let name = format!( + "{} ({})", + name, level + ); + diesel::insert_into(places) + .values(( + place_name.eq(&name), + slug.eq(&slugify( + &name, + )), + osm_id.eq(Some( + t_osm_id, + )), + osm_level + .eq(Some(level)), + )).get_result::(c) + }) + }).expect("Find or create place") + }; + if place.osm_id.is_none() { + debug!( + "Matched {:?} by name, update osm info", + place + ); + use schema::places::dsl::*; + diesel::update(places) + .filter(id.eq(place.id)) + .set(( + osm_id.eq(Some(t_osm_id)), + osm_level.eq(level), + )).execute(c) + .expect(&format!( + "Update OSM for {:?}", + place + )); + } + use models::PhotoPlace; + use schema::photo_places::dsl::*; + let q = photo_places + .filter(photo_id.eq(image)) + .filter(place_id.eq(place.id)); + if q.first::(c).is_ok() { + debug!( + "Photo #{} already has {:?}", + image, place.id + ); + } else { + diesel::insert_into(photo_places) + .values(( + photo_id.eq(image), + place_id.eq(place.id), + )).execute(c) + .expect("Place a photo"); + } + } + } + } + } else { + warn!("Bad response from overpass: {:?}", response); + } + } + Err(err) => { + warn!("Failed to get overpass info: {}", err); + } + } + + Ok(()) +} + +fn osm_id(obj: &Json) -> Option { + obj.find("id").and_then(|o| o.as_i64()) +} + +fn name_and_level(obj: &Json) -> Option<(&str, i16)> { + obj.find("tags").and_then(|tags| { + let name = tags + .find("name:sv") + //.or_else(|| tags.find("name:en")) + .or_else(|| tags.find("name")) + .and_then(|o| o.as_string()); + let level = tags + .find("admin_level") + .and_then(|o| o.as_string()) + .and_then(|s| s.parse().ok()); + if let (Some(name), Some(level)) = (name, level) { + Some((name, level)) + } else { + None + } + }) +} diff --git a/src/main.rs b/src/main.rs index d51458f..edd760f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,7 @@ extern crate typemap; mod adm; mod env; +mod fetch_places; mod memcachemiddleware; mod models; mod myexif; @@ -80,6 +81,14 @@ fn main() { .required(true) .help("Username to set password for"), ), + ).subcommand( + SubCommand::with_name("fetchplaces") + .about("Get place tags for photos by looking up coordinates in OSM") + .arg( + Arg::with_name("PHOTOS") + .required(true).multiple(true) + .help("Image ids to fetch place data for"), + ), ).subcommand( SubCommand::with_name("makepublic") .about("make specific image(s) public") @@ -172,6 +181,14 @@ fn run(args: &ArgMatches) -> Result<(), Error> { } ("stats", Some(_args)) => show_stats(&get_db()?), ("userlist", Some(_args)) => users::list(&get_db()?), + ("fetchplaces", Some(args)) => { + let db = get_db()?; + for photo in args.values_of("PHOTOS").unwrap() { + fetch_places::update_image_places(&db, photo.parse()?) + .map_err(|e| Error::Other(e))?; + } + Ok(()) + } ("userpass", Some(args)) => { users::passwd(&get_db()?, args.value_of("USER").unwrap()) } diff --git a/src/server/admin.rs b/src/server/admin.rs index 1f8ceec..7048b1f 100644 --- a/src/server/admin.rs +++ b/src/server/admin.rs @@ -8,8 +8,6 @@ use nickel::status::StatusCode; use nickel::{BodyError, FormBody, MiddlewareResult, Request, Response}; use nickel_diesel::DieselRequestExtensions; use nickel_jwt_session::SessionRequestExtensions; -use reqwest::Client; -use rustc_serialize::json::Json; use server::nickelext::MyResponse; use slug::slugify; @@ -244,105 +242,19 @@ pub fn fetch_places<'mw>( req: &mut Request, res: Response<'mw>, ) -> MiddlewareResult<'mw> { - use diesel; if !req.authorized_user().is_some() { return res.error(StatusCode::Unauthorized, "permission denied"); } let image = 60458; let c: &PgConnection = &req.db_conn(); - use schema::positions::dsl::*; - let coord = match positions - .filter(photo_id.eq(image)) - .select((latitude, longitude)) - .first::<(i32, i32)>(c) - { - Ok((tlat, tlong)) => Coord { - x: f64::from(tlat) / 1e6, - y: f64::from(tlong) / 1e6, - }, - Err(diesel::NotFound) => { - return res.not_found("Image has no position"); - } + use fetch_places::update_image_places; + match update_image_places(c, image) { + Ok(ok) => res.ok(|o| writeln!(o, "Ok, got places {:?}", ok)), Err(err) => { - error!("Failed to read position: {}", err); - return res.not_found("Failed to get image position"); - } - }; - info!("Should get places for {:?}", coord); - let client = Client::new(); - match client - .post("https://overpass.kumi.systems/api/interpreter") - .body(format!( - "[out:json];is_in({},{});area._[admin_level];out;", - coord.x, coord.y, - )).send() - { - Ok(mut response) => { - if response.status().is_success() { - let data = Json::from_reader(&mut response).unwrap(); - let obj = data.as_object().unwrap(); - if let Some(elements) = - obj.get("elements").and_then(|o| o.as_array()) - { - for obj in elements { - info!("{}", obj); - if let (Some(t_osm_id), Some((name, level))) = - (osm_id(obj), name_and_level(obj)) - { - info!("{}: {} (level {})", t_osm_id, name, level); - let place_id = { - // http://overpass-api.de/api/interpreter?data=%5Bout%3Acustom%5D%3Brel%5Bref%3D%22A+555%22%5D%5Bnetwork%3DBAB%5D%3Bout%3B - use models::Place; - use schema::places::dsl::*; - places - .filter(osm_id.eq(Some(t_osm_id))) - .first::(c) - .or_else(|_| { - diesel::insert_into(places) - .values(( - place_name.eq(&name), - slug.eq(&slugify(&name)), - osm_id.eq(Some(t_osm_id)), - osm_level.eq(Some(level)), - )).get_result::(c) - }).expect("Find or create tag") - }; - info!(" ...: {:?}", place_id) - } - } - } - } else { - warn!("Bad response from overpass: {:?}", response); - } - } - Err(err) => { - warn!("Failed to get overpass info: {}", err); + warn!("Failed to fetch places: {}", err); + // TODO This might be a not found or an internal server error + res.not_found("Failed to get image position") } } - - return res.ok(|o| writeln!(o, "Should get places for {:?}", coord)); -} - -fn osm_id(obj: &Json) -> Option { - obj.find("id").and_then(|o| o.as_i64()) -} - -fn name_and_level(obj: &Json) -> Option<(&str, i16)> { - obj.find("tags").and_then(|tags| { - let name = tags - .find("name:sv") - //.or_else(|| tags.find("name:en")) - .or_else(|| tags.find("name")) - .and_then(|o| o.as_string()); - let level = tags - .find("admin_level") - .and_then(|o| o.as_string()) - .and_then(|s| s.parse().ok()); - if let (Some(name), Some(level)) = (name, level) { - Some((name, level)) - } else { - None - } - }) }