andrey/album
1
0
Fork 0
Code Issues 10 Pull Requests Packages Projects Releases Wiki Activity

Refactor fetching places.

Browse Source 
Added a command line subcommand to fetch places for specified images.
This commit is contained in:
Rasmus Kaj 2018-09-05 23:39:35 +02:00
parent 0482c46b5a
commit 7cea04400f
5 changed files with 191 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
migrations/2018-09-05-065252_places_osm/down.sql
View File

@ -1,2 +1,4 @@
ALTER TABLE places DROP COLUMN osm_id;
ALTER TABLE places DROP COLUMN osm_level;
CREATE UNIQUE INDEX places_name_idx ON places (place_name);

3
migrations/2018-09-05-065252_places_osm/up.sql
View File

@ -3,4 +3,7 @@ ALTER TABLE places ADD COLUMN osm_id BIGINT UNIQUE;
ALTER TABLE places ADD COLUMN osm_level SMALLINT;
CREATE INDEX places_osm_idx ON places (osm_id);
CREATE INDEX places_osml_idx ON places (osm_level);
DROP INDEX places_name_idx;
CREATE UNIQUE INDEX places_name_idx ON places (place_name, osm_level);

163
src/fetch_places.rs Normal file
View File

@ -0,0 +1,163 @@
use diesel;
use diesel::prelude::*;
use models::Coord;
use reqwest::Client;
use rustc_serialize::json::Json;
use slug::slugify;
pub fn update_image_places(
c: &PgConnection,
image: i32,
) -> Result<(), String> {
use schema::positions::dsl::*;
let coord = match positions
.filter(photo_id.eq(image))
.select((latitude, longitude))
.first::<(i32, i32)>(c)
{
Ok((tlat, tlong)) => Coord {
x: f64::from(tlat) / 1e6,
y: f64::from(tlong) / 1e6,
},
Err(diesel::NotFound) => {
return Err(format!(
"Image #{} does not exist or has no position",
image
));
}
Err(err) => {
return Err(format!("Failed to get image position: {}", err));
}
};
debug!("Should get places for {:?}", coord);
let client = Client::new();
match client
.post("https://overpass.kumi.systems/api/interpreter")
.body(format!(
"[out:json];is_in({},{});area._[admin_level];out;",
coord.x, coord.y,
)).send()
{
Ok(mut response) => {
if response.status().is_success() {
let data = Json::from_reader(&mut response).unwrap();
let obj = data.as_object().unwrap();
if let Some(elements) =
obj.get("elements").and_then(|o| o.as_array())
{
for obj in elements {
if let (Some(t_osm_id), Some((name, level))) =
(osm_id(obj), name_and_level(obj))
{
debug!("{}: {} (level {})", t_osm_id, name, level);
let place = {
use models::Place;
use schema::places::dsl::*;
places
.filter(
osm_id.eq(Some(t_osm_id)).or(
place_name
.eq(name)
.and(osm_id.is_null()),
),
).first::<Place>(c)
.or_else(|_| {
diesel::insert_into(places)
.values((
place_name.eq(&name),
slug.eq(&slugify(&name)),
osm_id.eq(Some(t_osm_id)),
osm_level.eq(Some(level)),
)).get_result::<Place>(c)
.or_else(|_| {
let name = format!(
"{} ({})",
name, level
);
diesel::insert_into(places)
.values((
place_name.eq(&name),
slug.eq(&slugify(
&name,
)),
osm_id.eq(Some(
t_osm_id,
)),
osm_level
.eq(Some(level)),
)).get_result::<Place>(c)
})
}).expect("Find or create place")
};
if place.osm_id.is_none() {
debug!(
"Matched {:?} by name, update osm info",
place
);
use schema::places::dsl::*;
diesel::update(places)
.filter(id.eq(place.id))
.set((
osm_id.eq(Some(t_osm_id)),
osm_level.eq(level),
)).execute(c)
.expect(&format!(
"Update OSM for {:?}",
place
));
}
use models::PhotoPlace;
use schema::photo_places::dsl::*;
let q = photo_places
.filter(photo_id.eq(image))
.filter(place_id.eq(place.id));
if q.first::<PhotoPlace>(c).is_ok() {
debug!(
"Photo #{} already has {:?}",
image, place.id
);
} else {
diesel::insert_into(photo_places)
.values((
photo_id.eq(image),
place_id.eq(place.id),
)).execute(c)
.expect("Place a photo");
}
}
}
}
} else {
warn!("Bad response from overpass: {:?}", response);
}
}
Err(err) => {
warn!("Failed to get overpass info: {}", err);
}
}
Ok(())
}
fn osm_id(obj: &Json) -> Option<i64> {
obj.find("id").and_then(|o| o.as_i64())
}
fn name_and_level(obj: &Json) -> Option<(&str, i16)> {
obj.find("tags").and_then(|tags| {
let name = tags
.find("name:sv")
//.or_else(|| tags.find("name:en"))
.or_else(|| tags.find("name"))
.and_then(|o| o.as_string());
let level = tags
.find("admin_level")
.and_then(|o| o.as_string())
.and_then(|s| s.parse().ok());
if let (Some(name), Some(level)) = (name, level) {
Some((name, level))
} else {
None
}
})
}

17
src/main.rs
View File

@ -29,6 +29,7 @@ extern crate typemap;
mod adm;
mod env;
mod fetch_places;
mod memcachemiddleware;
mod models;
mod myexif;
@ -80,6 +81,14 @@ fn main() {
.required(true)
.help("Username to set password for"),
),
).subcommand(
SubCommand::with_name("fetchplaces")
.about("Get place tags for photos by looking up coordinates in OSM")
.arg(
Arg::with_name("PHOTOS")
.required(true).multiple(true)
.help("Image ids to fetch place data for"),
),
).subcommand(
SubCommand::with_name("makepublic")
.about("make specific image(s) public")
@ -172,6 +181,14 @@ fn run(args: &ArgMatches) -> Result<(), Error> {
}
("stats", Some(_args)) => show_stats(&get_db()?),
("userlist", Some(_args)) => users::list(&get_db()?),
("fetchplaces", Some(args)) => {
let db = get_db()?;
for photo in args.values_of("PHOTOS").unwrap() {
fetch_places::update_image_places(&db, photo.parse()?)
.map_err(|e| Error::Other(e))?;
}
Ok(())
}
("userpass", Some(args)) => {
users::passwd(&get_db()?, args.value_of("USER").unwrap())
}

100
src/server/admin.rs
View File

@ -8,8 +8,6 @@ use nickel::status::StatusCode;
use nickel::{BodyError, FormBody, MiddlewareResult, Request, Response};
use nickel_diesel::DieselRequestExtensions;
use nickel_jwt_session::SessionRequestExtensions;
use reqwest::Client;
use rustc_serialize::json::Json;
use server::nickelext::MyResponse;
use slug::slugify;
@ -244,105 +242,19 @@ pub fn fetch_places<'mw>(
req: &mut Request,
res: Response<'mw>,
) -> MiddlewareResult<'mw> {
use diesel;
if !req.authorized_user().is_some() {
return res.error(StatusCode::Unauthorized, "permission denied");
}
let image = 60458;
let c: &PgConnection = &req.db_conn();
use schema::positions::dsl::*;
let coord = match positions
.filter(photo_id.eq(image))
.select((latitude, longitude))
.first::<(i32, i32)>(c)
{
Ok((tlat, tlong)) => Coord {
x: f64::from(tlat) / 1e6,
y: f64::from(tlong) / 1e6,
},
Err(diesel::NotFound) => {
return res.not_found("Image has no position");
}
use fetch_places::update_image_places;
match update_image_places(c, image) {
Ok(ok) => res.ok(|o| writeln!(o, "Ok, got places {:?}", ok)),
Err(err) => {
error!("Failed to read position: {}", err);
return res.not_found("Failed to get image position");
}
};
info!("Should get places for {:?}", coord);
let client = Client::new();
match client
.post("https://overpass.kumi.systems/api/interpreter")
.body(format!(
"[out:json];is_in({},{});area._[admin_level];out;",
coord.x, coord.y,
)).send()
{
Ok(mut response) => {
if response.status().is_success() {
let data = Json::from_reader(&mut response).unwrap();
let obj = data.as_object().unwrap();
if let Some(elements) =
obj.get("elements").and_then(|o| o.as_array())
{
for obj in elements {
info!("{}", obj);
if let (Some(t_osm_id), Some((name, level))) =
(osm_id(obj), name_and_level(obj))
{
info!("{}: {} (level {})", t_osm_id, name, level);
let place_id = {
// http://overpass-api.de/api/interpreter?data=%5Bout%3Acustom%5D%3Brel%5Bref%3D%22A+555%22%5D%5Bnetwork%3DBAB%5D%3Bout%3B
use models::Place;
use schema::places::dsl::*;
places
.filter(osm_id.eq(Some(t_osm_id)))
.first::<Place>(c)
.or_else(|_| {
diesel::insert_into(places)
.values((
place_name.eq(&name),
slug.eq(&slugify(&name)),
osm_id.eq(Some(t_osm_id)),
osm_level.eq(Some(level)),
)).get_result::<Place>(c)
}).expect("Find or create tag")
};
info!(" ...: {:?}", place_id)
}
}
}
} else {
warn!("Bad response from overpass: {:?}", response);
}
}
Err(err) => {
warn!("Failed to get overpass info: {}", err);
warn!("Failed to fetch places: {}", err);
// TODO This might be a not found or an internal server error
res.not_found("Failed to get image position")
}
}
return res.ok(|o| writeln!(o, "Should get places for {:?}", coord));
}
fn osm_id(obj: &Json) -> Option<i64> {
obj.find("id").and_then(|o| o.as_i64())
}
fn name_and_level(obj: &Json) -> Option<(&str, i16)> {
obj.find("tags").and_then(|tags| {
let name = tags
.find("name:sv")
//.or_else(|| tags.find("name:en"))
.or_else(|| tags.find("name"))
.and_then(|o| o.as_string());
let level = tags
.find("admin_level")
.and_then(|o| o.as_string())
.and_then(|s| s.parse().ok());
if let (Some(name), Some(level)) = (name, level) {
Some((name, level))
} else {
None
}
})
}