use image::{DynamicImage, ImageFormat}; use std::cell::{Cell, RefCell}; use std::rc::Rc; use futures::channel::mpsc; use futures::prelude::*; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::*; use web_sys::*; mod history; mod ocr; // When the `wee_alloc` feature is enabled, use `wee_alloc` as the global // allocator. #[cfg(feature = "wee_alloc")] #[global_allocator] static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; async fn blob_to_image(blob: Blob) -> DynamicImage { let image_data = gloo::file::futures::read_as_bytes(&blob.into()) .await .unwrap(); image::load_from_memory_with_format(&image_data, ImageFormat::Png).unwrap() } fn image_to_data_uri(image: &DynamicImage) -> String { let mut result = String::from("data:image/png;base64,"); let mut enc = base64::write::EncoderStringWriter::from(&mut result, base64::STANDARD); image.write_to(&mut enc, ImageFormat::Png).unwrap(); let _ = enc.into_inner(); result } const QUEUE_SIZE_WIDTH: u32 = 70; const QUEUE_SIZE_HEIGHT: u32 = 20; fn get_queue_size(image: DynamicImage, corner: &(u32, u32)) -> DynamicImage { let &(x, y) = corner; let useful_region = image.crop_imm(x, y, QUEUE_SIZE_WIDTH, QUEUE_SIZE_HEIGHT); useful_region } fn update_target_marker( target_marker: &HtmlElement, width: &Rc>, height: &Rc>, video: &HtmlVideoElement, queue_position: &Rc>, ) { let (real_x, real_y) = queue_position.get(); let real_width = width.get() as f64; let real_height = height.get() as f64; let fake_width = video.offset_width() as f64; let fake_height = video.offset_height() as f64; let fake_x = real_x as f64 / real_width * fake_width; let fake_y = real_y as f64 / real_height * fake_height; let target_width = QUEUE_SIZE_WIDTH as f64 / real_width * fake_width; let target_height = QUEUE_SIZE_HEIGHT as f64 / real_height * fake_height; let style = target_marker.style(); style .set_property("left", &format!("{}px", fake_x)) .unwrap(); style.set_property("top", &format!("{}px", fake_y)).unwrap(); style .set_property("width", &format!("{}px", target_width)) .unwrap(); style .set_property("height", &format!("{}px", target_height)) .unwrap(); } async fn do_boot() { let width = Rc::new(Cell::new(0)); let height = Rc::new(Cell::new(0)); let streaming = Rc::new(Cell::new(false)); let queue_position = Rc::new(Cell::new((920u32, 520u32))); let document = gloo::utils::document(); let history = Rc::new(RefCell::new(history::History::default())); let (manual_update_tx, manual_update_rx) = mpsc::channel::<()>(5); let video: HtmlVideoElement = document .get_element_by_id("video") .unwrap() .dyn_into() .unwrap(); let canvas: HtmlCanvasElement = document .get_element_by_id("canvas") .unwrap() .dyn_into() .unwrap(); let target_marker: HtmlElement = document .get_element_by_id("target-marker") .unwrap() .dyn_into() .unwrap(); let video_click_listener = { let video = video.clone(); let also_video = video.clone(); let queue_position = queue_position.clone(); let width = width.clone(); let height = height.clone(); let target_marker = target_marker.clone(); let manual_update_tx = manual_update_tx.clone(); gloo::events::EventListener::new(&video, "click", move |event| { let mut manual_update_tx = manual_update_tx.clone(); let video = also_video.clone(); let mouse_event: &MouseEvent = event.dyn_ref().unwrap(); let mouse_x = mouse_event.offset_x() as f64; let mouse_y = mouse_event.offset_y() as f64; let real_width = width.get() as f64; let real_height = height.get() as f64; let fake_width = video.offset_width() as f64; let fake_height = video.offset_height() as f64; let real_x = mouse_x / fake_width * real_width - QUEUE_SIZE_WIDTH as f64 / 2.0; let real_y = mouse_y / fake_height * real_height - QUEUE_SIZE_HEIGHT as f64 / 2.0; queue_position.replace((real_x.round() as u32, real_y.round() as u32)); update_target_marker(&target_marker, &width, &height, &video, &queue_position); spawn_local(async move { manual_update_tx.send(()).await.unwrap() }); }) }; video_click_listener.forget(); let navigator = gloo::utils::window().navigator(); let media_devices = navigator.media_devices().unwrap(); let constraints = { let mut constraints = DisplayMediaStreamConstraints::new(); constraints.audio(&JsValue::FALSE); constraints.video(&JsValue::TRUE); constraints }; let capture_stream_promise = media_devices .get_display_media_with_constraints(&constraints) .unwrap(); let capture_stream = JsFuture::from(capture_stream_promise).await.unwrap(); let capture_stream: MediaStream = capture_stream.dyn_into().unwrap(); video.set_src_object(Some(&capture_stream)); let video_play_listener = { let width = width.clone(); let height = height.clone(); let streaming = streaming.clone(); let video = video.clone(); let also_video = video.clone(); let canvas = canvas.clone(); let queue_position = queue_position.clone(); let manual_update_tx = manual_update_tx.clone(); gloo::events::EventListener::new(&video, "canplay", move |_event| { let mut manual_update_tx = manual_update_tx.clone(); let video = also_video.clone(); let _ = video.play().unwrap(); if !streaming.get() { width.set(video.video_width()); height.set(video.video_height()); canvas.set_attribute("width", &format!("{}", width.get())); canvas.set_attribute("height", &format!("{}", height.get())); streaming.set(true); update_target_marker(&target_marker, &width, &height, &video, &queue_position); spawn_local(async move { manual_update_tx.send(()).await.unwrap() }); } }) }; video_play_listener.forget(); let update_stream = futures::stream_select!( gloo::timers::future::IntervalStream::new(2_000), manual_update_rx, ); let update_future = update_stream.for_each(move |_| { let history = history.clone(); let queue_position = queue_position.clone(); if !streaming.get() { return futures::future::ready(()); } let context = canvas.get_context("2d").unwrap().unwrap(); let context: CanvasRenderingContext2d = context.dyn_into().unwrap(); canvas.set_width(width.get()); canvas.set_height(height.get()); context .draw_image_with_html_video_element_and_dw_and_dh( &video, 0.0, 0.0, width.get().into(), height.get().into(), ) .unwrap(); canvas .to_blob( Closure::once_into_js(move |data: Blob| { let history = history.clone(); let queue_position = queue_position.clone(); // lifetime bullshit here let document = gloo::utils::document(); let photo = document.get_element_by_id("photo").unwrap(); let current_queue_position: HtmlElement = document .get_element_by_id("current-queue-position") .unwrap() .dyn_into() .unwrap(); let eta: HtmlElement = document .get_element_by_id("eta") .unwrap() .dyn_into() .unwrap(); spawn_local(async move { let image = blob_to_image(data).await; let useful_region = get_queue_size(image, &queue_position.get()); let data_uri = image_to_data_uri(&useful_region); photo.set_attribute("src", &data_uri).unwrap(); let queue_size = ocr::ocr(useful_region); match queue_size { Some(queue_size) => { current_queue_position.set_inner_text(&format!("{}", queue_size)); if let Ok(mut history) = history.try_borrow_mut() { history.record(queue_size); let estimated_finish = history.completion_time(); let eta_text = match estimated_finish { Some(finish) => { let empty_array = js_sys::Array::new(); let options = { let kv_pair = js_sys::Array::new(); kv_pair.push(&JsValue::from("timeStyle")); kv_pair.push(&JsValue::from("short")); let kv_pairs = js_sys::Array::new(); kv_pairs.push(&kv_pair); js_sys::Object::from_entries(&kv_pairs).unwrap() }; let format = js_sys::Intl::DateTimeFormat::new( &empty_array, &options, ); let format = format.format(); let locale_string = format.call1(&JsValue::UNDEFINED, &finish).unwrap(); locale_string.as_string().unwrap() } None => format!("pending"), }; let label = format!(" - ETA {}", eta_text); eta.set_inner_text(&label); } } None => { current_queue_position.set_inner_text("something i can't read"); } } }); }) .dyn_ref() .unwrap(), ) .unwrap(); futures::future::ready(()) }); spawn_local(update_future); } #[wasm_bindgen] pub fn boot() { console_error_panic_hook::set_once(); spawn_local(do_boot()); }