From 7b6a5e0e53a8dd2ca6d6f0e1af2b9af9f690e5bb Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Sun, 24 Mar 2024 14:10:00 +0100 Subject: [PATCH 01/10] CSV Import: Add upload page and format description to frontend --- frontend/src/routes/flights/+page.svelte | 5 +- frontend/src/routes/flights/FlightForm.svelte | 4 +- .../routes/flights/import/csv/+page.svelte | 249 ++++++++++++++++++ frontend/src/routes/flights/import/csv/api.ts | 46 ++++ frontend/src/routes/locations/+page.svelte | 2 +- frontend/static/example.csv | 8 + 6 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 frontend/src/routes/flights/import/csv/+page.svelte create mode 100644 frontend/src/routes/flights/import/csv/api.ts create mode 100644 frontend/static/example.csv diff --git a/frontend/src/routes/flights/+page.svelte b/frontend/src/routes/flights/+page.svelte index 830947c1..571a8d8e 100644 --- a/frontend/src/routes/flights/+page.svelte +++ b/frontend/src/routes/flights/+page.svelte @@ -134,9 +134,12 @@

You've logged {data.flights.length} flight{data.flights.length === 1 ? '' : 's'} so far!

-

+ +

Add flight + Import from CSV

+ diff --git a/frontend/src/routes/flights/FlightForm.svelte b/frontend/src/routes/flights/FlightForm.svelte index 81848d68..d56c1c34 100644 --- a/frontend/src/routes/flights/FlightForm.svelte +++ b/frontend/src/routes/flights/FlightForm.svelte @@ -31,8 +31,8 @@ // Form values // Note: Values for number inputs must allow null! - let files: FileList | undefined = undefined; - let igcBase64: string | undefined = undefined; + let files: FileList | undefined; + let igcBase64: string | undefined; let number: number | null = flight?.number ?? null; let glider: number | undefined = flight?.gliderId ?? lastGliderId; // Note: For the select input value binding to work correctly, entries from `locations` must be diff --git a/frontend/src/routes/flights/import/csv/+page.svelte b/frontend/src/routes/flights/import/csv/+page.svelte new file mode 100644 index 00000000..7e00e8ec --- /dev/null +++ b/frontend/src/routes/flights/import/csv/+page.svelte @@ -0,0 +1,249 @@ + + + + +{#if submitError?.type === 'authentication'} + +
+ Login +
+ +{:else if submitError?.type === 'api-error'} + (submitError = undefined)} + /> +{/if} + + + +

Import Flights from CSV

+ +
+
+  Warning: This import is still experimental. If you + experience any problems, please contact me at + flugbuech@bargen.dev! +
+
+ +
+
+

You can import a list of flights from a CSV file. It needs to follow this format:

+
    +
  • First row contains header fields
    • +
  • Character set: UTF-8
    • +
  • Delimiter: Comma (,)
    • +
  • Quoting: Double quotes (")
    • +
+
+ Click to expand full CSV format description +

CSV format description

+

+ The following header fields are supported (but they are all optional). All fields may be + omitted or empty (but at least one valid column must be present). +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescriptionExample
numberIntegerYour flight number108
dateISO StringThe date as ISO string2024-03-17
gliderString + The name of your glider/wing
+ + Must match the manufacturer and model of a glider you have already added to Flugbuech. + + 		Advance Xi 21
launch_siteString + The name of the launch site
+ Must match the name of a location you have already added to Flugbuech. + 		Ebenalp
launch_time_utcISO StringThe launch time (UTC!) as ISO time string13:37
landing_siteString + The name of the landing site
+ Must match the name of a location you have already added to Flugbuech. + 		Wasserauen
landing_time_utcISO StringThe landing time (UTC!) as ISO time string15:42
track_distanceFloatThe GPS track distance (in km) of your flight37.86
hikeandflyBooleanWas this a Hike&Fly? Either true or false.true
commentStringA comment about your flightWindy conditions, landed early
xcontest_urlStringLink to your flight on XContest + https://www.xcontest.org/world/en/flights/detail:dbrgn/7.3.2024/11:09 +
xcontest_tracktypeString + The track type of your flight on XContest
+ + Must be either free_flight, flat_triangle or + fai_triangle. + + 		flat_triangle
xcontest_scored_distanceFloatThe scored distance according to XContest36.79
video_urlStringLink to a video of your flighthttps://www.youtube.com/watch?v=PgyNx0V-hsU
+

You can find an example CSV file here.

+ + +
{ + event.preventDefault(); + void submitForm(); + }} + > +
+
+ +
+
+
+ +
+
+ + + diff --git a/frontend/src/routes/flights/import/csv/api.ts b/frontend/src/routes/flights/import/csv/api.ts new file mode 100644 index 00000000..69bfd5d7 --- /dev/null +++ b/frontend/src/routes/flights/import/csv/api.ts @@ -0,0 +1,46 @@ +import {z} from 'zod'; + +import {apiPostBlob, extractResponseError} from '$lib/api'; +import {AuthenticationError} from '$lib/errors'; + +const SCHEMA_API_ANALYZE_RESULT = z.object({ + warnings: z.array(z.string()).optional(), + errors: z.array(z.string()).optional(), +}); + +const SCHEMA_API_IMPORT_RESULT = z.object({ + success: z.boolean(), +}); + +type CsvAnalyzeResult = z.infer; +type CsvImportResult = z.infer; + +/** + * Analyze CSV file through API. + */ +export async function analyzeCsv(blob: Blob): Promise { + const res = await apiPostBlob('/api/v1/flights/add/csv/import?mode=analyze', blob); + switch (res.status) { + case 200: + return SCHEMA_API_ANALYZE_RESULT.parse(await res.json()); + case 401: + throw new AuthenticationError(); + default: + throw new Error(`Could not submit CSV to API: ${await extractResponseError(res)}`); + } +} + +/** + * Import CSV file through API. + */ +export async function importCsv(blob: Blob): Promise { + const res = await apiPostBlob('/api/v1/flights/add/csv/import?mode=import', blob); + switch (res.status) { + case 200: + return SCHEMA_API_IMPORT_RESULT.parse(await res.json()); + case 401: + throw new AuthenticationError(); + default: + throw new Error(`Could not submit CSV to API: ${await extractResponseError(res)}`); + } +} diff --git a/frontend/src/routes/locations/+page.svelte b/frontend/src/routes/locations/+page.svelte index acadac37..a26e804e 100644 --- a/frontend/src/routes/locations/+page.svelte +++ b/frontend/src/routes/locations/+page.svelte @@ -110,7 +110,7 @@
-  Note: A location can be used both as launch +  Note: A location can be used both as launch location and as landing location. Locations are not global, i.e. you are creating and maintaining your own location database.
diff --git a/frontend/static/example.csv b/frontend/static/example.csv new file mode 100644 index 00000000..e5eeec83 --- /dev/null +++ b/frontend/static/example.csv @@ -0,0 +1,8 @@ +number,date,glider,launch_site,launch_time_utc,landing_site,landing_time_utc,track_distance,hikeandfly,comment,xcontest_url,xcontest_tracktype,xcontest_scored_distance,video_url +13,2013-01-26,Team5 Green,Crap Sogn Gion,10:00,Larnags,10:12,4,,,,,, +232,2021-03-31,Advance Xi 21,"Fanas, Eggli",10:53,"Grüsch, Feld",14:19,126.63,false,,https://www.xcontest.org/switzerland/de/fluge/details:dbrgn/31.3.2021/10:53,flat_triangle,50.99, +262,2021-08-20,Advance Xi 21,Clariden,10:56,Planurahütte,11:10,7.69,true,"Aufstieg auf den Clariden ab Klausenpass. + +Wind von Südwest. Start ab Gipfel, anschliessend Soaring und Flug zur Planurahütte. + +Landung bei der Planurahütte auf dem Windkolk neben einem Gletscherflugzeug.",https://www.xcontest.org/2021/switzerland/de/fluge/details:dbrgn/20.8.2021/10:56,free_flight,3.20, From 6203a50765960d596df83606f6f92b39edf9be01 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Sun, 24 Mar 2024 21:48:17 +0100 Subject: [PATCH 02/10] CSV Import: Implement parsing and validation --- Cargo.lock | 22 + Cargo.toml | 1 + .../routes/flights/import/csv/+page.svelte | 9 +- frontend/src/routes/flights/import/csv/api.ts | 26 +- frontend/static/example.csv | 4 +- src/import_csv.rs | 753 ++++++++++++++++++ src/main.rs | 8 +- src/process_igc.rs | 4 +- src/responders/api_error.rs | 12 + src/test_utils.rs | 11 + src/xcontest.rs | 3 + 11 files changed, 839 insertions(+), 14 deletions(-) create mode 100644 src/import_csv.rs create mode 100644 src/xcontest.rs diff --git a/Cargo.lock b/Cargo.lock index fbb75ab9..0f014af0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -375,6 +375,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -580,6 +601,7 @@ dependencies = [ "base64 0.22.0", "chrono", "clap", + "csv", "diesel", "diesel-geography", "diesel_migrations", diff --git a/Cargo.toml b/Cargo.toml index 0a8cb4bb..cb959c2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } clap = "4" +csv = "1.3.0" diesel = { version = "2.1.4", features = ["postgres", "chrono"] } diesel-geography = { version = "0.2", features = ["serde"], git = "https://github.com/66np/diesel-geography", rev = "059c553" } diesel_migrations = { version = "2", features = ["postgres"] } diff --git a/frontend/src/routes/flights/import/csv/+page.svelte b/frontend/src/routes/flights/import/csv/+page.svelte index 7e00e8ec..07de2240 100644 --- a/frontend/src/routes/flights/import/csv/+page.svelte +++ b/frontend/src/routes/flights/import/csv/+page.svelte @@ -89,6 +89,7 @@
  • Character set: UTF-8
  • Delimiter: Comma (,)
  • Quoting: Double quotes (")
    • +
  • Max file size: 10 MiB
    • Click to expand full CSV format description @@ -139,8 +140,8 @@ launch_time_utc ISO String - The launch time (UTC!) as ISO time string - 13:37 + The launch time (UTC!) as ISO time string (including seconds) + 13:37:00 landing_site @@ -154,8 +155,8 @@ landing_time_utc ISO String - The landing time (UTC!) as ISO time string - 15:42 + The landing time (UTC!) as ISO time string (including seconds) + 15:42:23 track_distance diff --git a/frontend/src/routes/flights/import/csv/api.ts b/frontend/src/routes/flights/import/csv/api.ts index 69bfd5d7..2792344d 100644 --- a/frontend/src/routes/flights/import/csv/api.ts +++ b/frontend/src/routes/flights/import/csv/api.ts @@ -2,10 +2,28 @@ import {z} from 'zod'; import {apiPostBlob, extractResponseError} from '$lib/api'; import {AuthenticationError} from '$lib/errors'; +import {ensureXContestTracktype} from '$lib/xcontest'; + +const SCHEMA_API_CSV_FLIGHT_PREVIEW = z.object({ + number: z.number().optional(), + gliderId: z.number().optional(), + launchAt: z.number().optional(), + landingAt: z.number().optional(), + launchTime: z.string().optional(), + landingTime: z.string().optional(), + trackDistance: z.number().optional(), + xcontestTracktype: z.string().transform(ensureXContestTracktype).optional(), + xcontestDistance: z.number().optional(), + xcontestUrl: z.string().optional(), + comment: z.string().optional(), + videoUrl: z.string().optional(), + hikeandfly: z.boolean(), +}); const SCHEMA_API_ANALYZE_RESULT = z.object({ - warnings: z.array(z.string()).optional(), - errors: z.array(z.string()).optional(), + warnings: z.array(z.string()), + errors: z.array(z.string()), + flights: z.array(SCHEMA_API_CSV_FLIGHT_PREVIEW), }); const SCHEMA_API_IMPORT_RESULT = z.object({ @@ -19,7 +37,7 @@ type CsvImportResult = z.infer; * Analyze CSV file through API. */ export async function analyzeCsv(blob: Blob): Promise { - const res = await apiPostBlob('/api/v1/flights/add/csv/import?mode=analyze', blob); + const res = await apiPostBlob('/api/v1/flights/add/import_csv?mode=analyze', blob); switch (res.status) { case 200: return SCHEMA_API_ANALYZE_RESULT.parse(await res.json()); @@ -34,7 +52,7 @@ export async function analyzeCsv(blob: Blob): Promise { * Import CSV file through API. */ export async function importCsv(blob: Blob): Promise { - const res = await apiPostBlob('/api/v1/flights/add/csv/import?mode=import', blob); + const res = await apiPostBlob('/api/v1/flights/add/import_csv?mode=import', blob); switch (res.status) { case 200: return SCHEMA_API_IMPORT_RESULT.parse(await res.json()); diff --git a/frontend/static/example.csv b/frontend/static/example.csv index e5eeec83..41b4cb8d 100644 --- a/frontend/static/example.csv +++ b/frontend/static/example.csv @@ -1,6 +1,6 @@ number,date,glider,launch_site,launch_time_utc,landing_site,landing_time_utc,track_distance,hikeandfly,comment,xcontest_url,xcontest_tracktype,xcontest_scored_distance,video_url -13,2013-01-26,Team5 Green,Crap Sogn Gion,10:00,Larnags,10:12,4,,,,,, -232,2021-03-31,Advance Xi 21,"Fanas, Eggli",10:53,"Grüsch, Feld",14:19,126.63,false,,https://www.xcontest.org/switzerland/de/fluge/details:dbrgn/31.3.2021/10:53,flat_triangle,50.99, +13,2013-01-26,Team5 Green,Crap Sogn Gion,10:00:10,Larnags,10:12:13,4,,,,,, +232,2021-03-31,Advance Xi 21,"Fanas, Eggli",10:53:00,"Grüsch, Feld",14:19:00,126.63,false,,https://www.xcontest.org/switzerland/de/fluge/details:dbrgn/31.3.2021/10:53,flat_triangle,50.99, 262,2021-08-20,Advance Xi 21,Clariden,10:56,Planurahütte,11:10,7.69,true,"Aufstieg auf den Clariden ab Klausenpass. Wind von Südwest. Start ab Gipfel, anschliessend Soaring und Flug zur Planurahütte. diff --git a/src/import_csv.rs b/src/import_csv.rs new file mode 100644 index 00000000..9df1343e --- /dev/null +++ b/src/import_csv.rs @@ -0,0 +1,753 @@ +// API types + +use std::{collections::HashSet, io::Cursor}; + +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use diesel::PgConnection; +use log::{error, info}; +use rocket::{data::ToByteUnit, post, routes, serde::json::Json, Data, Route}; +use serde::{Deserialize, Serialize}; + +use crate::{auth, data, models::User, responders::ApiError, xcontest::is_valid_tracktype}; + +// API types + +#[derive(Debug, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CsvAnalyzeResult { + warnings: Vec, + errors: Vec, + flights: Vec, +} + +#[derive(Debug, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ApiCsvFlightPreview { + /// The user-defined flight number + #[serde(skip_serializing_if = "Option::is_none")] + pub number: Option, + /// The glider + #[serde(skip_serializing_if = "Option::is_none")] + pub glider_id: Option, + /// Launch location + #[serde(skip_serializing_if = "Option::is_none")] + pub launch_at: Option, + /// Landing location + #[serde(skip_serializing_if = "Option::is_none")] + pub landing_at: Option, + /// Time of launch + #[serde(skip_serializing_if = "Option::is_none")] + pub launch_time: Option>, + /// Time of landing + #[serde(skip_serializing_if = "Option::is_none")] + pub landing_time: Option>, + /// GPS track length + #[serde(skip_serializing_if = "Option::is_none")] + pub track_distance: Option, + /// XContest tracktype (free_flight, flat_triangle or fai_triangle) + #[serde(skip_serializing_if = "Option::is_none")] + pub xcontest_tracktype: Option, + /// XContest distance + #[serde(skip_serializing_if = "Option::is_none")] + pub xcontest_distance: Option, + /// XContest URL + #[serde(skip_serializing_if = "Option::is_none")] + pub xcontest_url: Option, + /// Comment your flight + #[serde(skip_serializing_if = "Option::is_none")] + pub comment: Option, + /// Link to a video of your flight + #[serde(skip_serializing_if = "Option::is_none")] + pub video_url: Option, + /// Whether you hiked up to launch + pub hikeandfly: bool, +} + +// Helper types + +static VALID_HEADERS: [&'static str; 14] = [ + "number", + "date", + "glider", + "launch_site", + "launch_time_utc", + "landing_site", + "landing_time_utc", + "track_distance", + "hikeandfly", + "comment", + "xcontest_url", + "xcontest_tracktype", + "xcontest_scored_distance", + "video_url", +]; + +#[derive(Debug, Deserialize)] +struct CsvRecord { + number: Option, + date: Option, + glider: Option, + launch_site: Option, + launch_time_utc: Option, + landing_site: Option, + landing_time_utc: Option, + track_distance: Option, + hikeandfly: Option, + comment: Option, + xcontest_url: Option, + xcontest_tracktype: Option, + xcontest_scored_distance: Option, + video_url: Option, +} + +// API endpoints + +/// Process a CSV file +/// +/// The `mode` GET parameter is required: +/// +/// - analyze: Process and analyze the CSV data, but don't store it yet +/// - import: Process and store CSV data +#[post( + "/flights/add/import_csv?", + format = "application/octet-stream", + data = "" +)] +pub async fn process_csv( + user: auth::AuthUser, + database: data::Database, + mode: &'_ str, + data: Data<'_>, +) -> Result, ApiError> { + info!("Processing CSV with mode '{mode}'"); + let user = user.into_inner(); + + // Validate mode + if !["analyze", "import"].contains(&mode) { + return Err(ApiError::InvalidData { + message: format!("Invalid mode: {mode}"), + }); + } + + // Right now the csv crate does not support async streams, so instead we'll collect the CSV data + // into a vec. This is fine, given that the max upload size is 10 MiB. + let csv_bytes = match data.open(crate::MAX_CSV_UPLOAD_BYTES.bytes()).into_bytes().await { + Ok(capped_bytes) => { + assert!( + capped_bytes.is_complete(), + "Expected capped bytes to be complete, but is_complete() returned false" + ); + capped_bytes.into_inner() + } + Err(e) => { + error!("Failed to read CSV data: {e:?}"); + return Err(ApiError::IoError { + message: format!("Failed to read CSV data"), + }); + } + }; + + // Process and analyze the CSV file + let analyze_result = database.run(move |db| analyze_csv(csv_bytes, &user, db)).await; + + if mode == "import" { + todo!("Import not yet implemented"); + } + + Ok(Json(analyze_result)) +} + +/// Return vec of all API routes. +pub fn api_routes() -> Vec { + routes![process_csv] +} + +// Helpers + +/// Process, analyze and return (but don't save) flights from CSV +fn analyze_csv(csv_bytes: Vec, user: &User, conn: &mut PgConnection) -> CsvAnalyzeResult { + let mut warnings = vec![]; + let mut errors = vec![]; + let mut flights = vec![]; + + macro_rules! fail { + ($msg:expr) => {{ + errors.push($msg); + return CsvAnalyzeResult { + warnings, + errors, + flights, + }; + }}; + } + + // Create CSV reader + let mut reader = csv::ReaderBuilder::new() + .has_headers(true) + .delimiter(b',') + .quote(b'"') + .from_reader(Cursor::new(csv_bytes)); + + // Parse and validate headers + let valid_header_set = HashSet::from(VALID_HEADERS); + match reader.headers() { + Ok(headers) => { + if headers.is_empty() { + fail!("CSV does not contain any columns".into()); + } + + let given_header_set = headers.into_iter().collect::>(); + if valid_header_set.is_disjoint(&given_header_set) { + fail!(format!( + "CSV header fields ({}) don't contain any valid header", + headers.iter().collect::>().join(","), + )); + } + } + Err(e) => fail!(format!("Error while reading headers from CSV: {e}")), + }; + + // Get user's gliders + let gliders: Vec<(i32, String)> = data::get_gliders_for_user(conn, user) + .into_iter() + .map(|glider| (glider.id, format!("{} {}", glider.manufacturer, glider.model))) + .collect(); + + // Get user's locations + let locations: Vec<(i32, String)> = data::get_locations_for_user(conn, user) + .into_iter() + .map(|location| (location.id, location.name)) + .collect(); + + // Parse and validate records + for (row_number, result) in reader.deserialize().enumerate() { + let record: CsvRecord = match result { + Ok(r) => r, + Err(e) => { + warnings.push(format!("Error while reading record from CSV: {e}")); + continue; + } + }; + let row_number1 = row_number + 1; + + // Prepare flight preview struct + let mut flight = ApiCsvFlightPreview { + number: record.number, + track_distance: record.track_distance, + comment: record.comment.clone(), + video_url: record.video_url.clone(), + hikeandfly: record.hikeandfly.unwrap_or(false), + ..Default::default() + }; + + flight_process_glider(&record, &mut flight, &gliders, &mut warnings); + flight_process_locations(&record, &mut flight, &locations, &mut warnings); + flight_process_date_time(&record, row_number1, &mut flight, &mut warnings); + flight_process_xcontest_info(&record, row_number1, &mut flight, &mut warnings); + + flights.push(flight); + } + + if flights.is_empty() { + fail!("CSV is empty".into()); + } + + CsvAnalyzeResult { + warnings, + errors, + flights, + } +} + +fn flight_process_glider( + record: &CsvRecord, + flight: &mut ApiCsvFlightPreview, + gliders: &Vec<(i32, String)>, + warnings: &mut Vec, +) { + if let Some(glider) = record.glider.as_ref() { + flight.glider_id = gliders + .iter() + .find_map(|(id, name)| if name == glider { Some(*id) } else { None }); + if flight.glider_id.is_none() { + warnings.push(format!( + "Could not find glider with name \"{glider}\" in your list of gliders" + )); + } + } +} + +fn flight_process_locations( + record: &CsvRecord, + flight: &mut ApiCsvFlightPreview, + locations: &Vec<(i32, String)>, + warnings: &mut Vec, +) { + // Look up launch and landing location + if let Some(launch_site) = record.launch_site.as_ref() { + flight.launch_at = locations + .iter() + .find_map(|(id, name)| if name == launch_site { Some(*id) } else { None }); + if flight.launch_at.is_none() { + warnings.push(format!( + "Could not find launch site with name \"{launch_site}\" in your list of locations" + )); + } + } + if let Some(landing_site) = record.landing_site.as_ref() { + flight.landing_at = locations + .iter() + .find_map(|(id, name)| if name == landing_site { Some(*id) } else { None }); + if flight.landing_at.is_none() { + warnings.push(format!( + "Could not find landing site with name \"{landing_site}\" in your list of locations" + )); + } + } +} + +fn flight_process_date_time( + record: &CsvRecord, + row_number1: usize, + flight: &mut ApiCsvFlightPreview, + warnings: &mut Vec, +) { + let date_parts = record.date.as_ref().map(|_| 1).unwrap_or_default() + + record.launch_time_utc.as_ref().map(|_| 1).unwrap_or_default() + + record.landing_time_utc.as_ref().map(|_| 1).unwrap_or_default(); + if date_parts > 0 && date_parts < 3 { + warnings.push(format!("Row {row_number1}: If you specify date, launch time or landing time, then the other two values must be provided as well")); + } + if let (Some(date), Some(launch_time), Some(landing_time)) = ( + record.date.as_ref().and_then(|date_str| { + NaiveDate::parse_from_str(date_str, "%Y-%m-%d") + .map_err(|_| warnings.push(format!("Row {row_number1}: Invalid ISO date: {}", date_str))) + .ok() + }), + record.launch_time_utc.as_ref().and_then(|time_str| { + NaiveTime::parse_from_str(time_str, "%H:%M:%S") + .map_err(|_| warnings.push(format!("Row {row_number1}: Invalid launch time: {}", time_str))) + .ok() + }), + record.landing_time_utc.as_ref().and_then(|time_str| { + NaiveTime::parse_from_str(time_str, "%H:%M:%S") + .map_err(|_| warnings.push(format!("Row {row_number1}: Invalid landing time: {}", time_str))) + .ok() + }), + ) { + flight.launch_time = Some(DateTime::from_naive_utc_and_offset( + NaiveDateTime::new(date, launch_time), + Utc, + )); + flight.landing_time = Some(DateTime::from_naive_utc_and_offset( + NaiveDateTime::new(date, landing_time), + Utc, + )); + } +} + +fn flight_process_xcontest_info( + record: &CsvRecord, + row_number1: usize, + flight: &mut ApiCsvFlightPreview, + warnings: &mut Vec, +) { + if let Some(tracktype) = record.xcontest_tracktype.as_ref() { + if !is_valid_tracktype(tracktype) { + warnings.push(format!( + "Row {row_number1}: Invalid XContest tracktype: {tracktype}" + )); + } else { + flight.xcontest_tracktype = Some(tracktype.into()); + } + } + flight.xcontest_distance = record.xcontest_scored_distance; + if let Some(url) = record.xcontest_url.as_ref() { + if url.starts_with("https://") { + flight.xcontest_url = Some(url.into()); + } else if url.starts_with("http://") { + flight.xcontest_url = Some(format!("https://{}", &url[7..])); + } else { + warnings.push(format!( + "Row {row_number1}: XContest URL must start with https:// or http://" + )); + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + models::{NewGlider, NewLocation}, + test_utils::{utc_datetime, DbTestContext}, + }; + + use super::*; + + fn analyze(csv: &'static str, ctx: Option) -> CsvAnalyzeResult { + let ctx = ctx.unwrap_or_else(|| DbTestContext::new()); + return analyze_csv( + csv.as_bytes().to_vec(), + &ctx.testuser1.user, + &mut ctx.force_get_conn(), + ); + } + + fn empty_vec() -> Vec { + vec![] + } + + #[test] + fn analyze_empty_csv() { + let result = analyze("", None); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, vec!["CSV does not contain any columns"]); + assert_eq!(result.flights, vec![]); + } + + #[test] + fn analyze_csv_without_valid_headers() { + let result = analyze("a,c,b\n1,2,3", None); + assert_eq!(result.warnings, empty_vec()); + assert_eq!( + result.errors, + vec!["CSV header fields (a,c,b) don't contain any valid header"] + ); + } + + #[test] + fn analyze_empty_csv_with_some_valid_headers() { + let result = analyze("a,number,b\n", None); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, vec!["CSV is empty"]); + } + + #[test] + fn analyze_csv_empty_glider() { + let result = analyze("number,glider\n42,", None); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].glider_id, None); + } + + #[test] + fn analyze_csv_unknown_glider() { + let result = analyze("number,glider\n42,Advance Omega ULS", None); + assert_eq!( + result.warnings, + vec!["Could not find glider with name \"Advance Omega ULS\" in your list of gliders"] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].glider_id, None); + } + + #[test] + fn analyze_csv_known_glider() { + let ctx = DbTestContext::new(); + let glider = data::create_glider( + &mut ctx.force_get_conn(), + NewGlider { + user_id: ctx.testuser1.user.id, + manufacturer: "Advance".into(), + model: "Omega ULS".into(), + since: None, + until: None, + source: None, + cost: None, + comment: None, + }, + ) + .unwrap(); + + let result = analyze("number,glider\n42,Advance Omega ULS", Some(ctx)); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].glider_id, Some(glider.id)); + } + + #[test] + fn analyze_csv_unknown_locations() { + let result = analyze("number,launch_site,landing_site\n42,Züri,Rappi", None); + assert_eq!( + result.warnings, + vec![ + "Could not find launch site with name \"Züri\" in your list of locations", + "Could not find landing site with name \"Rappi\" in your list of locations" + ] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].launch_at, None); + assert_eq!(result.flights[0].landing_at, None); + } + + #[test] + fn analyze_csv_known_locations() { + let ctx = DbTestContext::new(); + let location1 = data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Züri".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser1.user.id, + geog: None, + }, + ); + let location2 = data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Rappi".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser1.user.id, + geog: None, + }, + ); + + let result = analyze("number,launch_site,landing_site\n42,Züri,Rappi", Some(ctx)); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].launch_at, Some(location1.id)); + assert_eq!(result.flights[0].landing_at, Some(location2.id)); + } + + #[test] + fn analyze_csv_permission_checks() { + let ctx = DbTestContext::new(); + + // Note: All entities match but are owned by another user + data::create_glider( + &mut ctx.force_get_conn(), + NewGlider { + user_id: ctx.testuser2.user.id, + manufacturer: "Advance".into(), + model: "Alpha".into(), + since: None, + until: None, + source: None, + cost: None, + comment: None, + }, + ) + .unwrap(); + data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Züri".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser2.user.id, + geog: None, + }, + ); + data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Rappi".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser2.user.id, + geog: None, + }, + ); + + let result = analyze( + "number,glider,launch_site,landing_site\n42,Advance Alpha,Züri,Rappi", + Some(ctx), + ); + assert_eq!( + result.warnings, + vec![ + "Could not find glider with name \"Advance Alpha\" in your list of gliders", + "Could not find launch site with name \"Züri\" in your list of locations", + "Could not find landing site with name \"Rappi\" in your list of locations" + ] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].glider_id, None); + assert_eq!(result.flights[0].launch_at, None); + assert_eq!(result.flights[0].landing_at, None); + } + + #[test] + fn analyze_csv_partial_date_time() { + let result = analyze("number,date\n42,2023-12-12", None); + assert_eq!( + result.warnings, + vec![ + "Row 1: If you specify date, launch time or landing time, then the other two values must be provided as well" + ] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].launch_at, None); + assert_eq!(result.flights[0].landing_at, None); + } + + #[test] + fn analyze_csv_invalid_date_time() { + let result = analyze( + "number,date,launch_time_utc,landing_time_utc\n42,2023-13-44,asdf,2:15 pm", + None, + ); + assert_eq!( + result.warnings, + vec![ + "Row 1: Invalid ISO date: 2023-13-44", + "Row 1: Invalid launch time: asdf", + "Row 1: Invalid landing time: 2:15 pm", + ] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].launch_time, None); + assert_eq!(result.flights[0].landing_time, None); + } + + #[test] + fn analyze_csv_valid_date_time() { + let result = analyze( + "number,date,launch_time_utc,landing_time_utc\n42,2020-03-15,11:13:00,11:18:30", + None, + ); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!( + result.flights[0].launch_time, + Some(DateTime::from_timestamp(1584270780, 0).unwrap()) + ); + assert_eq!( + result.flights[0].landing_time, + Some(DateTime::from_timestamp(1584271110, 0).unwrap()) + ); + } + + #[test] + fn analyze_csv_invalid_xcontest_tracktype_url() { + let result = analyze( + "number,xcontest_tracktype,xcontest_url\n42,awesome_flight,xcontest.org/some/flight", + None, + ); + assert_eq!( + result.warnings, + vec![ + "Row 1: Invalid XContest tracktype: awesome_flight", + "Row 1: XContest URL must start with https:// or http://", + ] + ); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!(result.flights[0].xcontest_tracktype, None); + assert_eq!(result.flights[0].xcontest_url, None); + } + + #[test] + fn analyze_csv_map_xcontest_url_http() { + let result = analyze("number,xcontest_url\n42,http://xcontest.org/some/flight", None); + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + assert_eq!(result.flights[0].number, Some(42)); + assert_eq!( + result.flights[0].xcontest_url, + Some("https://xcontest.org/some/flight".into()) + ); + } + + #[test] + fn analyze_csv_full_example() { + let ctx = DbTestContext::new(); + + let glider = data::create_glider( + &mut ctx.force_get_conn(), + NewGlider { + user_id: ctx.testuser1.user.id, + manufacturer: "Advance".into(), + model: "Alpha".into(), + since: None, + until: None, + source: None, + cost: None, + comment: None, + }, + ) + .unwrap(); + let züri = data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Züri".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser1.user.id, + geog: None, + }, + ); + let rappi = data::create_location( + &mut ctx.force_get_conn(), + NewLocation { + name: "Rappi".into(), + country: "CH".into(), + elevation: 0, + user_id: ctx.testuser1.user.id, + geog: None, + }, + ); + + let result = analyze( + "number,date,glider,launch_site,launch_time_utc,landing_site,landing_time_utc,track_distance,hikeandfly,comment,xcontest_url,xcontest_tracktype,xcontest_scored_distance,video_url\n\ + 1,2020-01-01,Advance Alpha,Züri,10:00:00,Rappi,11:00:00,44.7,,\"Some flying, some scratching\",https://xcontest.org/myflight/,free_flight,27,https://youtube.com/myvid\n\ + 2,2020-01-02,Advance Alpha,Rappi,12:01:02,Züri,14:50:50,50,true,Way back,,,,\n\ + ,,,,,,,,false,,,,,\n", + Some(ctx)); + + assert_eq!(result.warnings, empty_vec()); + assert_eq!(result.errors, empty_vec()); + + assert_eq!( + result.flights[0], + ApiCsvFlightPreview { + number: Some(1), + glider_id: Some(glider.id), + launch_at: Some(züri.id), + landing_at: Some(rappi.id), + launch_time: Some(utc_datetime(2020, 1, 1, 10, 0, 0)), + landing_time: Some(utc_datetime(2020, 1, 1, 11, 0, 0)), + track_distance: Some(44.7), + xcontest_tracktype: Some("free_flight".into()), + xcontest_distance: Some(27.0), + xcontest_url: Some("https://xcontest.org/myflight/".into()), + comment: Some("Some flying, some scratching".into()), + video_url: Some("https://youtube.com/myvid".into()), + hikeandfly: false + } + ); + assert_eq!( + result.flights[1], + ApiCsvFlightPreview { + number: Some(2), + glider_id: Some(glider.id), + launch_at: Some(rappi.id), + landing_at: Some(züri.id), + launch_time: Some(utc_datetime(2020, 1, 2, 12, 1, 2)), + landing_time: Some(utc_datetime(2020, 1, 2, 14, 50, 50)), + track_distance: Some(50.0), + comment: Some("Way back".into()), + hikeandfly: true, + ..Default::default() + } + ); + assert_eq!( + result.flights[2], + ApiCsvFlightPreview { + hikeandfly: false, + ..Default::default() + } + ); + } +} diff --git a/src/main.rs b/src/main.rs index c99be064..7e73bfd7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ mod cors; mod data; mod flights; mod gliders; +mod import_csv; mod locations; mod models; mod process_igc; @@ -18,13 +19,17 @@ mod schema; mod stats; #[cfg(test)] mod test_utils; +mod xcontest; use anyhow::{Context, Result}; use clap::{Arg, ArgAction, Command}; use rocket::{catch, catchers, get, request::Request, routes}; use serde::Deserialize; -pub const MAX_UPLOAD_BYTES: u64 = 50 * 1024 * 1024; +// Limits +pub const MAX_IGC_UPLOAD_BYTES: u64 = 50 * 1024 * 1024; +pub const MAX_CSV_UPLOAD_BYTES: u64 = 10 * 1024 * 1024; + pub const NAME: &str = "flugbuech-api"; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const DESCRIPTION: &str = "Paragliding flight book."; @@ -114,6 +119,7 @@ async fn main() -> Result<()> { gliders::api_routes(), flights::api_routes(), process_igc::api_routes(), + import_csv::api_routes(), ] .concat(), ); diff --git a/src/process_igc.rs b/src/process_igc.rs index 0090b60a..5925b52d 100644 --- a/src/process_igc.rs +++ b/src/process_igc.rs @@ -211,8 +211,6 @@ fn parse_igc(reader: impl BufRead, user: &models::User, db: &mut diesel::PgConne } /// Process IGC file, return parsed data. -/// -/// This endpoint is meant to be called from a fetch request. #[post( "/flights/add/process_igc", format = "application/octet-stream", @@ -226,7 +224,7 @@ pub async fn process_igc( let user = user.into_inner(); // Open IGC file - let igc_bytes = match data.open(crate::MAX_UPLOAD_BYTES.bytes()).into_bytes().await { + let igc_bytes = match data.open(crate::MAX_IGC_UPLOAD_BYTES.bytes()).into_bytes().await { Ok(capped_vec) if capped_vec.is_complete() => capped_vec.into_inner(), Ok(_) => { return Json(FlightInfoResult::Error { diff --git a/src/responders/api_error.rs b/src/responders/api_error.rs index 8dcc6ba9..5b458df3 100644 --- a/src/responders/api_error.rs +++ b/src/responders/api_error.rs @@ -44,6 +44,7 @@ impl RocketError { pub enum ApiError { MissingAuthentication, InvalidData { message: String }, + IoError { message: String }, NotFound, } @@ -73,6 +74,17 @@ impl<'r> response::Responder<'r, 'static> for ApiError { }) .unwrap(), ), + ApiError::IoError { message } => ( + Status::InternalServerError, + json::to_string(&RocketError { + error: RocketErrorInner { + code: 500, + reason: "IoError", + description: message.into(), + }, + }) + .unwrap(), + ), ApiError::NotFound => ( Status::NotFound, json::to_string(&RocketError { diff --git a/src/test_utils.rs b/src/test_utils.rs index 411a0064..ce562e59 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -4,6 +4,7 @@ use std::{ sync::{Mutex, MutexGuard}, }; +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; use diesel::{connection::SimpleConnection, pg::PgConnection, prelude::*}; use diesel_migrations::MigrationHarness; use dotenv; @@ -131,3 +132,13 @@ pub fn make_test_config() -> rocket::figment::Figment { .select(Config::DEBUG_PROFILE) .merge(("databases.flugbuech", database)) } + +pub fn utc_datetime(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> DateTime { + DateTime::from_naive_utc_and_offset( + NaiveDateTime::new( + NaiveDate::from_ymd_opt(year, month, day).unwrap(), + NaiveTime::from_hms_opt(hour, min, sec).unwrap(), + ), + Utc, + ) +} diff --git a/src/xcontest.rs b/src/xcontest.rs new file mode 100644 index 00000000..ede2b1ef --- /dev/null +++ b/src/xcontest.rs @@ -0,0 +1,3 @@ +pub fn is_valid_tracktype(value: &str) -> bool { + ["free_flight", "flat_triangle", "fai_triangle"].contains(&value) +} From e5540ab73ad5e80a5bb120a4a5c0334c34f5e4f6 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Sun, 24 Mar 2024 21:55:02 +0100 Subject: [PATCH 03/10] CSV Import: Warn on unknown fields --- src/import_csv.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/import_csv.rs b/src/import_csv.rs index 9df1343e..25dd4670 100644 --- a/src/import_csv.rs +++ b/src/import_csv.rs @@ -203,6 +203,17 @@ fn analyze_csv(csv_bytes: Vec, user: &User, conn: &mut PgConnection) -> CsvA headers.iter().collect::>().join(","), )); } + let mut unknown_fields = given_header_set + .difference(&valid_header_set) + .cloned() + .collect::>(); + if !unknown_fields.is_empty() { + unknown_fields.sort(); + warnings.push(format!( + "Some CSV header fields are unknown and will be ignored: {}", + unknown_fields.join(",") + )); + } } Err(e) => fail!(format!("Error while reading headers from CSV: {e}")), }; @@ -417,8 +428,11 @@ mod tests { #[test] fn analyze_empty_csv_with_some_valid_headers() { - let result = analyze("a,number,b\n", None); - assert_eq!(result.warnings, empty_vec()); + let result = analyze("a,number,c,b\n", None); + assert_eq!( + result.warnings, + vec!["Some CSV header fields are unknown and will be ignored: a,b,c"], + ); assert_eq!(result.errors, vec!["CSV is empty"]); } From e4f8754433aa16639b81341701ca6506ef3f0960 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Sun, 31 Mar 2024 20:28:19 +0200 Subject: [PATCH 04/10] CSV Import: Add error context, show analyze result --- frontend/src/lib/formatters.ts | 7 + .../routes/flights/import/csv/+page.svelte | 402 +++++++++++------- frontend/src/routes/flights/import/csv/api.ts | 24 +- src/import_csv.rs | 231 +++++++--- 4 files changed, 451 insertions(+), 213 deletions(-) diff --git a/frontend/src/lib/formatters.ts b/frontend/src/lib/formatters.ts index 246fbe25..154dba14 100644 --- a/frontend/src/lib/formatters.ts +++ b/frontend/src/lib/formatters.ts @@ -25,6 +25,13 @@ export function formatTime(time: Date): string { return time.toISOString().slice(11, 16); } +/** + * Format a datetime as "YYYY-mm-dd hh:mm". + */ +export function formatDateTime(datetime: Date): string { + return `${formatDate(datetime)} ${formatTime(datetime)}`; +} + /** * Format a distance in km. * diff --git a/frontend/src/routes/flights/import/csv/+page.svelte b/frontend/src/routes/flights/import/csv/+page.svelte index 07de2240..6ab61c90 100644 --- a/frontend/src/routes/flights/import/csv/+page.svelte +++ b/frontend/src/routes/flights/import/csv/+page.svelte @@ -1,10 +1,11 @@